Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,23 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+205123/wandb/run-20241210_205130-svysfc56/files/media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4 filter=lfs diff=lfs merge=lfs -text
+205123/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+205451/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+205451/wandb/run-20241210_205457-52qyo5rz/files/media/videos/eval/video_0_bfb699cd8c28a00c2c34.mp4 filter=lfs diff=lfs merge=lfs -text
+205451/episode_rosbags/episode_0_2024-12-10-20-55-17.bag filter=lfs diff=lfs merge=lfs -text
+214032/episode_rosbags/episode_2_2024-12-10-21-42-51.rrd filter=lfs diff=lfs merge=lfs -text
+214032/episode_rosbags/episode_0_2024-12-10-21-41-00.rrd filter=lfs diff=lfs merge=lfs -text
+214032/episode_rosbags/episode_1_2024-12-10-21-41-44.rrd filter=lfs diff=lfs merge=lfs -text
+214032/episode_rosbags/episode_4_2024-12-10-21-44-14.rrd filter=lfs diff=lfs merge=lfs -text
+214032/wandb/run-20241210_214038-s9hpsjf9/run-s9hpsjf9.wandb filter=lfs diff=lfs merge=lfs -text
+214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/4_eval_4_86ec182da7a62ff7b578.mp4 filter=lfs diff=lfs merge=lfs -text
+214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/0_eval_0_99cde89795fdf8f2318a.mp4 filter=lfs diff=lfs merge=lfs -text
+214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/2_eval_2_aac838f24394620216df.mp4 filter=lfs diff=lfs merge=lfs -text
+214032/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/1_eval_1_0f1a83ed932af73a96be.mp4 filter=lfs diff=lfs merge=lfs -text
+214032/wandb/run-20241210_214038-s9hpsjf9/files/media/videos/eval/3_eval_3_9f3ddf93c395db4f718a.mp4 filter=lfs diff=lfs merge=lfs -text
+214032/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+214032/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+214032/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+214032/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text

204954/.hydra/config.yaml

@@ -0,0 +1,349 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '265626'
+true_action_history: true
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

204954/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${experiment_dir}/${now:%H%M%S}
+  sweep:
+    dir: ${experiment_dir}/${now:%H%M%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${experiment_dir}/${now:%H%M%S}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_policy
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/204954
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

204954/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion

204954/eval_policy.log

@@ -0,0 +1,15 @@
+[2024-12-10 20:49:54,427][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-10 20:49:54,430][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2024-12-10 20:49:55,731][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+

205123/.hydra/config.yaml

@@ -0,0 +1,349 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '265626'
+true_action_history: true
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

205123/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${experiment_dir}/${now:%H%M%S}
+  sweep:
+    dir: ${experiment_dir}/${now:%H%M%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${experiment_dir}/${now:%H%M%S}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

205123/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion

205123/episode_rosbags/aligned_depth_to_color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

205123/episode_rosbags/cam_tf_world.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256

205123/episode_rosbags/color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

205123/episode_rosbags/depth_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200

205123/eval_robot.log

@@ -0,0 +1,12 @@
+[2024-12-10 20:51:23,588][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-10 20:51:23,591][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+

205123/eval_video/0_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6b0b3ba17b7513a5df9bca8084b0eda15d2f82df085783449312a47400bba10
+size 942048

205123/tb/events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c19a2cb159a40df6149928fa963c721dd4fb49916ec660df1f61fffc949c49ec
+size 88

205123/wandb/debug-internal.log

@@ -0,0 +1,351 @@
+2024-12-10 20:51:30,311 INFO    StreamThr :597014 [internal.py:wandb_internal():85] W&B internal server running at pid: 597014, started at: 2024-12-10 20:51:30.311157
+2024-12-10 20:51:30,312 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status
+2024-12-10 20:51:30,325 INFO    WriterThread:597014 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/run-svysfc56.wandb
+2024-12-10 20:51:30,325 DEBUG   SenderThread:597014 [sender.py:send():379] send: header
+2024-12-10 20:51:30,336 DEBUG   SenderThread:597014 [sender.py:send():379] send: run
+2024-12-10 20:51:30,645 INFO    SenderThread:597014 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files
+2024-12-10 20:51:30,645 INFO    SenderThread:597014 [sender.py:_start_run_threads():1188] run started: svysfc56 with start time 1733881890.312299
+2024-12-10 20:51:30,656 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: check_version
+2024-12-10 20:51:30,656 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: check_version
+2024-12-10 20:51:30,743 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: run_start
+2024-12-10 20:51:30,748 DEBUG   HandlerThread:597014 [system_info.py:__init__():26] System info init
+2024-12-10 20:51:30,748 DEBUG   HandlerThread:597014 [system_info.py:__init__():41] System info init done
+2024-12-10 20:51:30,748 INFO    HandlerThread:597014 [system_monitor.py:start():194] Starting system monitor
+2024-12-10 20:51:30,748 INFO    SystemMonitor:597014 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-10 20:51:30,749 INFO    HandlerThread:597014 [system_monitor.py:probe():214] Collecting system info
+2024-12-10 20:51:30,749 INFO    SystemMonitor:597014 [interfaces.py:start():188] Started cpu monitoring
+2024-12-10 20:51:30,750 INFO    SystemMonitor:597014 [interfaces.py:start():188] Started disk monitoring
+2024-12-10 20:51:30,750 INFO    SystemMonitor:597014 [interfaces.py:start():188] Started gpu monitoring
+2024-12-10 20:51:30,751 INFO    SystemMonitor:597014 [interfaces.py:start():188] Started memory monitoring
+2024-12-10 20:51:30,751 INFO    SystemMonitor:597014 [interfaces.py:start():188] Started network monitoring
+2024-12-10 20:51:30,761 DEBUG   HandlerThread:597014 [system_info.py:probe():152] Probing system
+2024-12-10 20:51:30,762 DEBUG   HandlerThread:597014 [system_info.py:_probe_git():137] Probing git
+2024-12-10 20:51:30,767 DEBUG   HandlerThread:597014 [system_info.py:_probe_git():145] Probing git done
+2024-12-10 20:51:30,767 DEBUG   HandlerThread:597014 [system_info.py:probe():200] Probing system done
+2024-12-10 20:51:30,767 DEBUG   HandlerThread:597014 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-11T01:51:30.761221', 'startedAt': '2024-12-11T01:51:30.302896', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '4f7c0421eb84a7caf6bf5670f68777523362676f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4124.7015, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.948, 'min': 800.0, 'max': 4700.0}, {'current': 4499.997, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 364.1780319213867}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71596908569336}}
+2024-12-10 20:51:30,768 INFO    HandlerThread:597014 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-10 20:51:30,768 INFO    HandlerThread:597014 [system_monitor.py:probe():227] Publishing system info
+2024-12-10 20:51:30,768 DEBUG   HandlerThread:597014 [system_info.py:_save_code():44] Saving code
+2024-12-10 20:51:30,769 DEBUG   HandlerThread:597014 [system_info.py:_save_code():67] Saving code done
+2024-12-10 20:51:30,769 DEBUG   HandlerThread:597014 [system_info.py:_save_patches():84] Saving git patches
+2024-12-10 20:51:30,832 DEBUG   HandlerThread:597014 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-10 20:51:30,832 INFO    HandlerThread:597014 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-10 20:51:30,836 DEBUG   SenderThread:597014 [sender.py:send():379] send: files
+2024-12-10 20:51:30,836 INFO    SenderThread:597014 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-10 20:51:30,836 INFO    SenderThread:597014 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-10 20:51:30,836 INFO    SenderThread:597014 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-10 20:51:31,124 INFO    wandb-upload_2:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/tiz5ae9w-diff.patch
+2024-12-10 20:51:31,126 INFO    wandb-upload_0:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/p6o167mp-wandb-metadata.json
+2024-12-10 20:51:31,143 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-10 20:51:31,144 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: python_packages
+2024-12-10 20:51:31,145 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-10 20:51:31,145 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: stop_status
+2024-12-10 20:51:31,146 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:31,147 INFO    HandlerThread:597014 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/tb"
+  save: true
+}
+control {
+  relay_id: "42f7013eb6ef4a01b21dd778abc8c686"
+}
+_info {
+  stream_id: "svysfc56"
+}
+
+2024-12-10 20:51:31,147 DEBUG   HandlerThread:597014 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-10 20:51:31,173 INFO    wandb-upload_1:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/9ln43j5e-code/FISH/eval_robot.py
+2024-12-10 20:51:31,218 DEBUG   SenderThread:597014 [sender.py:send():379] send: telemetry
+2024-12-10 20:51:31,218 DEBUG   SenderThread:597014 [sender.py:send():379] send: tbrecord
+2024-12-10 20:51:31,218 DEBUG   SenderThread:597014 [sender.py:send():379] send: files
+2024-12-10 20:51:31,218 INFO    SenderThread:597014 [sender.py:_save_file():1454] saving file events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0 with policy live
+2024-12-10 20:51:31,415 INFO    wandb-upload_2:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/08emxf8k-events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0
+2024-12-10 20:51:31,647 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/diff.patch
+2024-12-10 20:51:31,647 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/code/FISH/eval_robot.py
+2024-12-10 20:51:31,647 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/requirements.txt
+2024-12-10 20:51:31,647 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:51:31,647 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-metadata.json
+2024-12-10 20:51:31,647 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0
+2024-12-10 20:51:31,647 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/code/FISH
+2024-12-10 20:51:31,647 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/code
+2024-12-10 20:51:32,144 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:33,145 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:33,646 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:51:34,145 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:35,145 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:36,144 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:51:36,145 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:37,145 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:38,145 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:39,145 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:40,145 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:41,144 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:51:41,146 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:42,146 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:43,146 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:44,146 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:45,146 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:46,144 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-10 20:51:46,144 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: stop_status
+2024-12-10 20:51:46,186 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:46,300 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:51:46,908 DEBUG   SenderThread:597014 [sender.py:send():379] send: config
+2024-12-10 20:51:47,146 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:48,146 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:49,146 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:49,651 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:51:50,146 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:51,147 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:51,335 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:51:52,147 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:53,147 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:53,652 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:51:54,147 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:55,147 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:56,147 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:57,145 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:51:57,147 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:57,654 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:51:58,148 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:59,148 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:00,148 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:01,144 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-10 20:52:01,144 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: stop_status
+2024-12-10 20:52:01,189 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:02,148 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:02,300 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:52:02,655 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/config.yaml
+2024-12-10 20:52:03,148 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:04,149 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:05,149 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:06,182 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:07,149 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:07,657 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:08,149 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:08,194 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:52:09,149 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:09,657 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:10,149 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:11,149 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:11,658 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:12,149 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:13,149 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:13,219 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:52:13,659 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:14,150 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:15,150 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:15,659 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:16,144 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-10 20:52:16,144 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: stop_status
+2024-12-10 20:52:16,150 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:17,150 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:17,660 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:18,150 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:18,404 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:52:19,152 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:19,661 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:20,152 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:21,152 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:21,661 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:22,152 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:23,152 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:23,407 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:52:23,662 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:24,152 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:25,153 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:25,662 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:26,153 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:27,153 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:27,663 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:28,153 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:28,607 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:52:29,153 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:29,664 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:30,154 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:30,752 DEBUG   SystemMonitor:597014 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-10 20:52:30,752 DEBUG   SenderThread:597014 [sender.py:send():379] send: stats
+2024-12-10 20:52:31,144 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-10 20:52:31,144 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: stop_status
+2024-12-10 20:52:31,154 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:31,666 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:32,154 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:33,154 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:33,667 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:33,908 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:52:34,154 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:34,667 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:35,154 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:35,668 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:36,154 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:37,155 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:38,155 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:38,668 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:39,155 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:39,371 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:52:40,155 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:41,155 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:42,155 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:43,155 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:44,155 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:44,374 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-10 20:52:44,375 DEBUG   SenderThread:597014 [sender.py:send():379] send: files
+2024-12-10 20:52:44,375 INFO    SenderThread:597014 [sender.py:_save_file():1454] saving file media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4 with policy now
+2024-12-10 20:52:44,375 DEBUG   SenderThread:597014 [sender.py:send():379] send: history
+2024-12-10 20:52:44,375 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:52:44,375 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: summary_record
+2024-12-10 20:52:44,377 INFO    SenderThread:597014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-10 20:52:44,377 DEBUG   SenderThread:597014 [sender.py:send():379] send: exit
+2024-12-10 20:52:44,377 INFO    SenderThread:597014 [sender.py:send_exit():586] handling exit code: 0
+2024-12-10 20:52:44,377 INFO    SenderThread:597014 [sender.py:send_exit():588] handling runtime: 73
+2024-12-10 20:52:44,377 INFO    SenderThread:597014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-10 20:52:44,377 INFO    SenderThread:597014 [sender.py:send_exit():594] send defer
+2024-12-10 20:52:44,379 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:44,379 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-10 20:52:44,380 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:44,380 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-10 20:52:44,380 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 1
+2024-12-10 20:52:44,380 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:44,380 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-10 20:52:44,380 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:44,380 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-10 20:52:44,380 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 2
+2024-12-10 20:52:44,380 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:44,380 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-10 20:52:44,380 INFO    HandlerThread:597014 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-10 20:52:44,381 DEBUG   SystemMonitor:597014 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-10 20:52:44,381 DEBUG   SystemMonitor:597014 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-10 20:52:44,381 INFO    HandlerThread:597014 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-10 20:52:44,382 INFO    HandlerThread:597014 [interfaces.py:finish():200] Joined disk monitor
+2024-12-10 20:52:44,439 INFO    HandlerThread:597014 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-10 20:52:44,439 INFO    HandlerThread:597014 [interfaces.py:finish():200] Joined memory monitor
+2024-12-10 20:52:44,439 INFO    HandlerThread:597014 [interfaces.py:finish():200] Joined network monitor
+2024-12-10 20:52:44,440 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:44,440 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-10 20:52:44,440 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 3
+2024-12-10 20:52:44,440 DEBUG   SenderThread:597014 [sender.py:send():379] send: stats
+2024-12-10 20:52:44,440 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:44,440 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-10 20:52:44,440 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:44,440 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-10 20:52:44,440 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 4
+2024-12-10 20:52:44,441 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:44,441 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-10 20:52:44,670 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-summary.json
+2024-12-10 20:52:44,671 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4
+2024-12-10 20:52:44,671 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/media/videos/eval
+2024-12-10 20:52:44,671 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/media/videos
+2024-12-10 20:52:44,671 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/media
+2024-12-10 20:52:44,696 INFO    wandb-upload_1:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/6w704qfx-media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4
+2024-12-10 20:52:45,030 INFO    wandb-upload_2:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/kk0w0e5k-media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4
+2024-12-10 20:52:45,671 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:51,156 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-10 20:52:51,156 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:52:51,156 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-10 20:52:51,157 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:51,157 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-10 20:52:51,157 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 5
+2024-12-10 20:52:51,157 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: poll_exit
+2024-12-10 20:52:51,157 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:51,157 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-10 20:52:51,157 DEBUG   SenderThread:597014 [sender.py:send():379] send: summary
+2024-12-10 20:52:51,158 INFO    SenderThread:597014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-10 20:52:51,158 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:51,158 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-10 20:52:51,158 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 6
+2024-12-10 20:52:51,158 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:51,158 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-10 20:52:51,158 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:51,158 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-10 20:52:51,158 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 7
+2024-12-10 20:52:51,158 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:52:51,158 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:51,158 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-10 20:52:51,159 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:51,159 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-10 20:52:51,375 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-10 20:52:51,673 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-summary.json
+2024-12-10 20:52:51,921 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 8
+2024-12-10 20:52:51,922 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: poll_exit
+2024-12-10 20:52:51,922 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:51,922 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-10 20:52:51,922 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:51,922 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-10 20:52:51,922 INFO    SenderThread:597014 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-10 20:52:51,924 INFO    SenderThread:597014 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-10 20:52:51,942 INFO    SenderThread:597014 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-10 20:52:51,951 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 9
+2024-12-10 20:52:51,952 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:51,952 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-10 20:52:51,952 DEBUG   SenderThread:597014 [sender.py:send():379] send: artifact
+2024-12-10 20:52:52,376 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-10 20:52:52,626 INFO    wandb-upload_0:597014 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmptoffctew
+2024-12-10 20:52:52,673 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:52,858 INFO    wandb-upload_2:597014 [upload_job.py:push():88] Uploaded file /tmp/tmpemx2d4yl/wandb-job.json
+2024-12-10 20:52:52,880 INFO    wandb-upload_1:597014 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpc2y7tffk
+2024-12-10 20:52:53,418 INFO    SenderThread:597014 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM2Njc1OTg1Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM1MTcyNjQ1OA==', 'versionIndex': 335}}}
+2024-12-10 20:52:53,418 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:53,418 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-10 20:52:53,418 INFO    SenderThread:597014 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-10 20:52:53,674 INFO    SenderThread:597014 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files
+2024-12-10 20:52:53,674 INFO    SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/config.yaml config.yaml
+2024-12-10 20:52:53,674 INFO    SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-summary.json wandb-summary.json
+2024-12-10 20:52:53,674 INFO    SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/requirements.txt requirements.txt
+2024-12-10 20:52:53,674 INFO    SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-metadata.json wandb-metadata.json
+2024-12-10 20:52:53,674 INFO    SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0 events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0
+2024-12-10 20:52:53,674 INFO    SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log output.log
+2024-12-10 20:52:53,676 INFO    SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/diff.patch diff.patch
+2024-12-10 20:52:53,680 INFO    SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-10 20:52:53,680 INFO    SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4 media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4
+2024-12-10 20:52:53,680 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 10
+2024-12-10 20:52:53,680 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: poll_exit
+2024-12-10 20:52:53,681 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:53,681 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-10 20:52:53,681 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:53,681 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-10 20:52:53,681 INFO    SenderThread:597014 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-10 20:52:53,872 INFO    wandb-upload_2:597014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/requirements.txt
+2024-12-10 20:52:53,888 INFO    wandb-upload_0:597014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/config.yaml
+2024-12-10 20:52:53,899 INFO    wandb-upload_3:597014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-summary.json
+2024-12-10 20:52:53,910 INFO    wandb-upload_1:597014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:54,110 INFO    Thread-11 (_thread_body):597014 [sender.py:transition_state():614] send defer: 11
+2024-12-10 20:52:54,110 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:54,111 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-10 20:52:54,111 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:54,111 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-10 20:52:54,111 INFO    SenderThread:597014 [file_pusher.py:join():175] waiting for file pusher
+2024-12-10 20:52:54,111 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 12
+2024-12-10 20:52:54,111 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:54,111 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-10 20:52:54,112 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:54,112 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-10 20:52:54,112 INFO    SenderThread:597014 [file_stream.py:finish():601] file stream finish called
+2024-12-10 20:52:54,317 INFO    SenderThread:597014 [file_stream.py:finish():605] file stream finish is done
+2024-12-10 20:52:54,317 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 13
+2024-12-10 20:52:54,318 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:54,318 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-10 20:52:54,318 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:54,318 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-10 20:52:54,318 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 14
+2024-12-10 20:52:54,318 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:54,318 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-10 20:52:54,319 DEBUG   SenderThread:597014 [sender.py:send():379] send: final
+2024-12-10 20:52:54,319 DEBUG   SenderThread:597014 [sender.py:send():379] send: footer
+2024-12-10 20:52:54,319 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:54,319 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-10 20:52:54,319 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-10 20:52:54,320 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: poll_exit
+2024-12-10 20:52:54,320 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: server_info
+2024-12-10 20:52:54,320 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: server_info
+2024-12-10 20:52:54,322 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-10 20:52:54,322 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-10 20:52:54,322 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-10 20:52:54,322 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:54,378 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: poll_exit
+2024-12-10 20:52:54,378 INFO    MainThread:597014 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-10 20:52:54,378 INFO    MainThread:597014 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-10 20:52:54,379 INFO    MainThread:597014 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-10 20:52:54,379 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-10 20:52:54,379 INFO    HandlerThread:597014 [handler.py:finish():882] shutting down handler
+2024-12-10 20:52:55,322 INFO    WriterThread:597014 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/run-svysfc56.wandb
+2024-12-10 20:52:55,378 INFO    SenderThread:597014 [sender.py:finish():1615] shutting down sender
+2024-12-10 20:52:55,378 INFO    SenderThread:597014 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-10 20:52:55,378 INFO    SenderThread:597014 [file_pusher.py:join():175] waiting for file pusher

205123/wandb/debug.log

@@ -0,0 +1,31 @@
+2024-12-10 20:51:30,303 INFO    MainThread:596892 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-10 20:51:30,303 INFO    MainThread:596892 [wandb_setup.py:_flush():76] Configure stats pid to 596892
+2024-12-10 20:51:30,303 INFO    MainThread:596892 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-10 20:51:30,303 INFO    MainThread:596892 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/settings
+2024-12-10 20:51:30,303 INFO    MainThread:596892 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-10 20:51:30,303 INFO    MainThread:596892 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-10 20:51:30,304 INFO    MainThread:596892 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-10 20:51:30,304 INFO    MainThread:596892 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-10 20:51:30,304 INFO    MainThread:596892 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/logs/debug.log
+2024-12-10 20:51:30,304 INFO    MainThread:596892 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/logs/debug-internal.log
+2024-12-10 20:51:30,304 INFO    MainThread:596892 [wandb_init.py:init():569] calling init triggers
+2024-12-10 20:51:30,304 INFO    MainThread:596892 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '265626', 'wandb_notes': '265626_req_265827', 'eval': True, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim_left_to_right'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'obs_shape': [5, 240, 320], 'action_shape': [36, 7], 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}, 'true_action_history': True}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'true_action_history': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt'}
+2024-12-10 20:51:30,304 INFO    MainThread:596892 [wandb_init.py:init():619] starting backend
+2024-12-10 20:51:30,304 INFO    MainThread:596892 [wandb_init.py:init():623] setting up manager
+2024-12-10 20:51:30,310 INFO    MainThread:596892 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-10 20:51:30,312 INFO    MainThread:596892 [wandb_init.py:init():631] backend started and connected
+2024-12-10 20:51:30,324 INFO    MainThread:596892 [wandb_init.py:init():720] updated telemetry
+2024-12-10 20:51:30,334 INFO    MainThread:596892 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-10 20:51:30,656 INFO    MainThread:596892 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-10 20:51:30,737 INFO    MainThread:596892 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.0 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-10 20:51:30,737 INFO    MainThread:596892 [wandb_init.py:init():804] starting run threads in backend
+2024-12-10 20:51:31,144 INFO    MainThread:596892 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-10 20:51:31,145 INFO    MainThread:596892 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-10 20:51:31,145 INFO    MainThread:596892 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-10 20:51:31,145 INFO    MainThread:596892 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-10 20:51:31,147 INFO    MainThread:596892 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-10 20:51:31,147 INFO    MainThread:596892 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/tb, True
+2024-12-10 20:51:46,908 INFO    MainThread:596892 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'}
+2024-12-10 20:52:55,379 WARNING MsgRouterThr:596892 [router.py:message_loop():77] message_loop has been closed

205123/wandb/run-20241210_205130-svysfc56/files/code/FISH/eval_robot.py

@@ -0,0 +1,512 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)		
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()

205123/wandb/run-20241210_205130-svysfc56/files/config.yaml

@@ -0,0 +1,966 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 0
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '265626'
+wandb_notes:
+  desc: null
+  value: 265626_req_265827
+eval:
+  desc: null
+  value: true
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: true
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: false
+use_sdf_maps:
+  desc: null
+  value: false
+use_normals_maps:
+  desc: null
+  value: false
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value:
+  - hbm_twodim_left_to_right
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 0
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/265626
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    obs_shape:
+    - 5
+    - 240
+    - 320
+    action_shape:
+    - 36
+    - 7
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: false
+        use_sdf_maps: false
+        use_normals_maps: false
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+      policy_frequency: 5
+      interpolation_frequency: 25
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 0
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: false
+        use_sdf_maps: false
+        use_normals_maps: false
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: true
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: false
+          use_sdf_maps: false
+          use_normals_maps: false
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+        policy_frequency: 5
+        interpolation_frequency: 25
+      true_action_history: true
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1733881890
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 16
+      - 23
+      - 35
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: hbm
+left_book_slot:
+  desc: null
+  value: twodim

205123/wandb/run-20241210_205130-svysfc56/files/diff.patch

@@ -0,0 +1,25 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index a9568e3..c42ff82 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -123,6 +123,11 @@ bc_weight_type: 'qfilter' # linear, qfilter
+ 
+ # Load weights
+ load_checkpoint: ${agent.load_checkpoint}
++# 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
++# full dataset
++wandb_run_id: "265626" # RGBD+Mask+act history
++# wandb_run_id: "265628" # RGBD+Mask+contact+act history
++
+ # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # full dataset
+ # 6 step action history
+@@ -135,7 +140,7 @@ load_checkpoint: ${agent.load_checkpoint}
+ # wandb_run_id: '264772_0' # RGBD+act history
+ # wandb_run_id: '264775_0' # RGBD+mask+act history
+ # wandb_run_id: '264776_0' # RGBD+contact+act history
+-wandb_run_id: '264777_0' # RGBD+mask+contact+act history
++# wandb_run_id: '264777_0' # RGBD+mask+contact+act history
+ 
+ # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # 16 held out demos

205123/wandb/run-20241210_205130-svysfc56/files/media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6b0b3ba17b7513a5df9bca8084b0eda15d2f82df085783449312a47400bba10
+size 942048

205123/wandb/run-20241210_205130-svysfc56/files/output.log

@@ -0,0 +1,1049 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1733881906.908934]: resetting environment
+[INFO] [1733881906.913976]: cleared current plan
+[INFO] [1733881906.915491]: moving to home
+[INFO] [1733881908.619161]: reached home
+[INFO] [1733881908.620458]: reset action history
+[INFO] [1733881911.334703]: environment reset
+Starting episode 0
+[INFO] [1733881911.338240]: resetting environment
+[INFO] [1733881911.346125]: cleared current plan
+[INFO] [1733881911.347839]: moving to home
+[INFO] [1733881912.349890]: reached home
+[INFO] [1733881912.351231]: reset action history
+[INFO] [1733881915.058954]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1733881926.140345]: resetting environment
+[INFO] [1733881926.145656]: cleared current plan
+[INFO] [1733881926.150768]: moving to home
+[INFO] [1733881927.153098]: reached home
+[INFO] [1733881927.154883]: reset action history
+[INFO] [1733881929.863341]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/episode_rosbags/episode_0_2024-12-10-20-52-09.bag
+... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-596892.log
+started roslaunch server http://158.130.50.37:44627/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [597118]
+process[print_text-2]: started with pid [597119]
+process[rosbag_record-3]: started with pid [597143]
+started bagging!
+[INFO] [1733881930.601407]: received mask_obs with header seq: 1
+stamp:
+  secs: 1733881930
+  nsecs: 598197937
+frame_id: '' and grasped_object_mask header seq: 7743
+stamp:
+  secs: 1733881930
+  nsecs: 373498678
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881930.790134]: received mask_obs with header seq: 2
+stamp:
+  secs: 1733881930
+  nsecs: 787627696
+frame_id: '' and grasped_object_mask header seq: 7749
+stamp:
+  secs: 1733881930
+  nsecs: 573334217
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881930.989544]: received mask_obs with header seq: 3
+stamp:
+  secs: 1733881930
+  nsecs: 987669944
+frame_id: '' and grasped_object_mask header seq: 7755
+stamp:
+  secs: 1733881930
+  nsecs: 773170948
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881931.186965]: received mask_obs with header seq: 4
+stamp:
+  secs: 1733881931
+  nsecs: 185687780
+frame_id: '' and grasped_object_mask header seq: 7761
+stamp:
+  secs: 1733881930
+  nsecs: 973006487
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881931.405436]: received mask_obs with header seq: 5
+stamp:
+  secs: 1733881931
+  nsecs: 404456853
+frame_id: '' and grasped_object_mask header seq: 7767
+stamp:
+  secs: 1733881931
+  nsecs: 172842264
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881931.606265]: received mask_obs with header seq: 6
+stamp:
+  secs: 1733881931
+  nsecs: 605320453
+frame_id: '' and grasped_object_mask header seq: 7773
+stamp:
+  secs: 1733881931
+  nsecs: 372677803
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881931.810085]: received mask_obs with header seq: 7
+stamp:
+  secs: 1733881931
+  nsecs: 809148073
+frame_id: '' and grasped_object_mask header seq: 7779
+stamp:
+  secs: 1733881931
+  nsecs: 572512865
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881932.000967]: received mask_obs with header seq: 8
+stamp:
+  secs: 1733881932
+  nsecs:     77486
+frame_id: '' and grasped_object_mask header seq: 7785
+stamp:
+  secs: 1733881931
+  nsecs: 772349596
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881932.200750]: received mask_obs with header seq: 9
+stamp:
+  secs: 1733881932
+  nsecs: 199798583
+frame_id: '' and grasped_object_mask header seq: 7791
+stamp:
+  secs: 1733881931
+  nsecs: 972187757
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881932.404178]: received mask_obs with header seq: 10
+stamp:
+  secs: 1733881932
+  nsecs: 403140544
+frame_id: '' and grasped_object_mask header seq: 7797
+stamp:
+  secs: 1733881932
+  nsecs: 172024965
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881932.603047]: received mask_obs with header seq: 11
+stamp:
+  secs: 1733881932
+  nsecs: 602175712
+frame_id: '' and grasped_object_mask header seq: 7803
+stamp:
+  secs: 1733881932
+  nsecs: 371861935
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881932.802278]: received mask_obs with header seq: 12
+stamp:
+  secs: 1733881932
+  nsecs: 801258563
+frame_id: '' and grasped_object_mask header seq: 7809
+stamp:
+  secs: 1733881932
+  nsecs: 571698666
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881933.018084]: received mask_obs with header seq: 13
+stamp:
+  secs: 1733881933
+  nsecs:  17174005
+frame_id: '' and grasped_object_mask header seq: 7815
+stamp:
+  secs: 1733881932
+  nsecs: 771533251
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881933.218519]: received mask_obs with header seq: 14
+stamp:
+  secs: 1733881933
+  nsecs: 217555284
+frame_id: '' and grasped_object_mask header seq: 7821
+stamp:
+  secs: 1733881932
+  nsecs: 971366405
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881933.411083]: received mask_obs with header seq: 15
+stamp:
+  secs: 1733881933
+  nsecs: 410149812
+frame_id: '' and grasped_object_mask header seq: 7827
+stamp:
+  secs: 1733881933
+  nsecs: 171200514
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881933.616644]: received mask_obs with header seq: 16
+stamp:
+  secs: 1733881933
+  nsecs: 615602254
+frame_id: '' and grasped_object_mask header seq: 7833
+stamp:
+  secs: 1733881933
+  nsecs: 371033192
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881933.810750]: received mask_obs with header seq: 17
+stamp:
+  secs: 1733881933
+  nsecs: 809497833
+frame_id: '' and grasped_object_mask header seq: 7839
+stamp:
+  secs: 1733881933
+  nsecs: 570865870
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881934.011081]: received mask_obs with header seq: 18
+stamp:
+  secs: 1733881934
+  nsecs:  10175704
+frame_id: '' and grasped_object_mask header seq: 7845
+stamp:
+  secs: 1733881933
+  nsecs: 770703554
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881934.204773]: received mask_obs with header seq: 19
+stamp:
+  secs: 1733881934
+  nsecs: 203893184
+frame_id: '' and grasped_object_mask header seq: 7851
+stamp:
+  secs: 1733881933
+  nsecs: 970544338
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881934.433582]: received mask_obs with header seq: 20
+stamp:
+  secs: 1733881934
+  nsecs: 432651519
+frame_id: '' and grasped_object_mask header seq: 7857
+stamp:
+  secs: 1733881934
+  nsecs: 170385599
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881934.628506]: received mask_obs with header seq: 21
+stamp:
+  secs: 1733881934
+  nsecs: 626872062
+frame_id: '' and grasped_object_mask header seq: 7864
+stamp:
+  secs: 1733881934
+  nsecs: 403534174
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881934.826157]: received mask_obs with header seq: 22
+stamp:
+  secs: 1733881934
+  nsecs: 825274705
+frame_id: '' and grasped_object_mask header seq: 7870
+stamp:
+  secs: 1733881934
+  nsecs: 603375435
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881935.031275]: received mask_obs with header seq: 23
+stamp:
+  secs: 1733881935
+  nsecs:  29773235
+frame_id: '' and grasped_object_mask header seq: 7876
+stamp:
+  secs: 1733881934
+  nsecs: 803215981
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881935.228458]: received mask_obs with header seq: 24
+stamp:
+  secs: 1733881935
+  nsecs: 227554321
+frame_id: '' and grasped_object_mask header seq: 7882
+stamp:
+  secs: 1733881935
+  nsecs:   3057957
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881935.422727]: received mask_obs with header seq: 25
+stamp:
+  secs: 1733881935
+  nsecs: 421752691
+frame_id: '' and grasped_object_mask header seq: 7887
+stamp:
+  secs: 1733881935
+  nsecs: 169591427
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881935.638195]: received mask_obs with header seq: 26
+stamp:
+  secs: 1733881935
+  nsecs: 637306213
+frame_id: '' and grasped_object_mask header seq: 7894
+stamp:
+  secs: 1733881935
+  nsecs: 402739286
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881935.837922]: received mask_obs with header seq: 27
+stamp:
+  secs: 1733881935
+  nsecs: 837003231
+frame_id: '' and grasped_object_mask header seq: 7900
+stamp:
+  secs: 1733881935
+  nsecs: 602580309
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881936.034971]: received mask_obs with header seq: 28
+stamp:
+  secs: 1733881936
+  nsecs:  33885240
+frame_id: '' and grasped_object_mask header seq: 7906
+stamp:
+  secs: 1733881935
+  nsecs: 802418470
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881936.223721]: received mask_obs with header seq: 29
+stamp:
+  secs: 1733881936
+  nsecs: 222499608
+frame_id: '' and grasped_object_mask header seq: 7912
+stamp:
+  secs: 1733881936
+  nsecs:   2255917
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881936.446193]: received mask_obs with header seq: 30
+stamp:
+  secs: 1733881936
+  nsecs: 445185184
+frame_id: '' and grasped_object_mask header seq: 7918
+stamp:
+  secs: 1733881936
+  nsecs: 202094316
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881936.632934]: received mask_obs with header seq: 31
+stamp:
+  secs: 1733881936
+  nsecs: 631910800
+frame_id: '' and grasped_object_mask header seq: 7924
+stamp:
+  secs: 1733881936
+  nsecs: 401931524
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881936.838232]: received mask_obs with header seq: 32
+stamp:
+  secs: 1733881936
+  nsecs: 837055444
+frame_id: '' and grasped_object_mask header seq: 7930
+stamp:
+  secs: 1733881936
+  nsecs: 601768732
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881937.032954]: received mask_obs with header seq: 33
+stamp:
+  secs: 1733881937
+  nsecs:  32030105
+frame_id: '' and grasped_object_mask header seq: 7936
+stamp:
+  secs: 1733881936
+  nsecs: 801601171
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881937.227681]: received mask_obs with header seq: 34
+stamp:
+  secs: 1733881937
+  nsecs: 225735902
+frame_id: '' and grasped_object_mask header seq: 7942
+stamp:
+  secs: 1733881937
+  nsecs:   1431465
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881937.424884]: received mask_obs with header seq: 35
+stamp:
+  secs: 1733881937
+  nsecs: 423624753
+frame_id: '' and grasped_object_mask header seq: 7948
+stamp:
+  secs: 1733881937
+  nsecs: 201261759
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881937.602132]: received mask_obs with header seq: 36
+stamp:
+  secs: 1733881937
+  nsecs: 601197004
+frame_id: '' and grasped_object_mask header seq: 7953
+stamp:
+  secs: 1733881937
+  nsecs: 367787123
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881937.819760]: received mask_obs with header seq: 37
+stamp:
+  secs: 1733881937
+  nsecs: 817212343
+frame_id: '' and grasped_object_mask header seq: 7959
+stamp:
+  secs: 1733881937
+  nsecs: 567616940
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881938.023777]: received mask_obs with header seq: 38
+stamp:
+  secs: 1733881938
+  nsecs:  22439956
+frame_id: '' and grasped_object_mask header seq: 7965
+stamp:
+  secs: 1733881937
+  nsecs: 767450571
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881938.203714]: received mask_obs with header seq: 39
+stamp:
+  secs: 1733881938
+  nsecs: 202681779
+frame_id: '' and grasped_object_mask header seq: 7971
+stamp:
+  secs: 1733881937
+  nsecs: 967287064
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881938.403625]: received mask_obs with header seq: 40
+stamp:
+  secs: 1733881938
+  nsecs: 402268409
+frame_id: '' and grasped_object_mask header seq: 7977
+stamp:
+  secs: 1733881938
+  nsecs: 167124510
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881938.600104]: received mask_obs with header seq: 41
+stamp:
+  secs: 1733881938
+  nsecs: 598839759
+frame_id: '' and grasped_object_mask header seq: 7983
+stamp:
+  secs: 1733881938
+  nsecs: 366960764
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881938.820781]: received mask_obs with header seq: 42
+stamp:
+  secs: 1733881938
+  nsecs: 819800138
+frame_id: '' and grasped_object_mask header seq: 7989
+stamp:
+  secs: 1733881938
+  nsecs: 566796780
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881939.014330]: received mask_obs with header seq: 43
+stamp:
+  secs: 1733881939
+  nsecs:  13423442
+frame_id: '' and grasped_object_mask header seq: 7996
+stamp:
+  secs: 1733881938
+  nsecs: 799937010
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881939.224791]: received mask_obs with header seq: 44
+stamp:
+  secs: 1733881939
+  nsecs: 223258733
+frame_id: '' and grasped_object_mask header seq: 8001
+stamp:
+  secs: 1733881938
+  nsecs: 966463566
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881939.425529]: received mask_obs with header seq: 45
+stamp:
+  secs: 1733881939
+  nsecs: 424565792
+frame_id: '' and grasped_object_mask header seq: 8008
+stamp:
+  secs: 1733881939
+  nsecs: 199602365
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881939.619431]: received mask_obs with header seq: 46
+stamp:
+  secs: 1733881939
+  nsecs: 618312597
+frame_id: '' and grasped_object_mask header seq: 8014
+stamp:
+  secs: 1733881939
+  nsecs: 399434566
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881939.820555]: received mask_obs with header seq: 47
+stamp:
+  secs: 1733881939
+  nsecs: 819616794
+frame_id: '' and grasped_object_mask header seq: 8020
+stamp:
+  secs: 1733881939
+  nsecs: 599266529
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881940.005593]: received mask_obs with header seq: 48
+stamp:
+  secs: 1733881940
+  nsecs:   4656076
+frame_id: '' and grasped_object_mask header seq: 8025
+stamp:
+  secs: 1733881939
+  nsecs: 765801430
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881940.204921]: received mask_obs with header seq: 49
+stamp:
+  secs: 1733881940
+  nsecs: 203764915
+frame_id: '' and grasped_object_mask header seq: 8031
+stamp:
+  secs: 1733881939
+  nsecs: 965646029
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881940.413592]: received mask_obs with header seq: 50
+stamp:
+  secs: 1733881940
+  nsecs: 412583351
+frame_id: '' and grasped_object_mask header seq: 8037
+stamp:
+  secs: 1733881940
+  nsecs: 165490627
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881940.618829]: received mask_obs with header seq: 51
+stamp:
+  secs: 1733881940
+  nsecs: 617256879
+frame_id: '' and grasped_object_mask header seq: 8043
+stamp:
+  secs: 1733881940
+  nsecs: 365335941
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881940.824649]: received mask_obs with header seq: 52
+stamp:
+  secs: 1733881940
+  nsecs: 823161363
+frame_id: '' and grasped_object_mask header seq: 8050
+stamp:
+  secs: 1733881940
+  nsecs: 598489761
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881941.027861]: received mask_obs with header seq: 53
+stamp:
+  secs: 1733881941
+  nsecs:  26394367
+frame_id: '' and grasped_object_mask header seq: 8056
+stamp:
+  secs: 1733881940
+  nsecs: 798328876
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881941.225607]: received mask_obs with header seq: 54
+stamp:
+  secs: 1733881941
+  nsecs: 224196434
+frame_id: '' and grasped_object_mask header seq: 8061
+stamp:
+  secs: 1733881940
+  nsecs: 964860439
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881941.403943]: received mask_obs with header seq: 55
+stamp:
+  secs: 1733881941
+  nsecs: 402922868
+frame_id: '' and grasped_object_mask header seq: 8067
+stamp:
+  secs: 1733881941
+  nsecs: 164697409
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881941.609296]: received mask_obs with header seq: 56
+stamp:
+  secs: 1733881941
+  nsecs: 608402252
+frame_id: '' and grasped_object_mask header seq: 8073
+stamp:
+  secs: 1733881941
+  nsecs: 364534378
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881941.801054]: received mask_obs with header seq: 57
+stamp:
+  secs: 1733881941
+  nsecs: 799741744
+frame_id: '' and grasped_object_mask header seq: 8079
+stamp:
+  secs: 1733881941
+  nsecs: 564371347
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881942.001835]: received mask_obs with header seq: 58
+stamp:
+  secs: 1733881942
+  nsecs:    708103
+frame_id: '' and grasped_object_mask header seq: 8085
+stamp:
+  secs: 1733881941
+  nsecs: 764206648
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881942.199729]: received mask_obs with header seq: 59
+stamp:
+  secs: 1733881942
+  nsecs: 198785305
+frame_id: '' and grasped_object_mask header seq: 8091
+stamp:
+  secs: 1733881941
+  nsecs: 964039087
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881942.426365]: received mask_obs with header seq: 60
+stamp:
+  secs: 1733881942
+  nsecs: 425239324
+frame_id: '' and grasped_object_mask header seq: 8097
+stamp:
+  secs: 1733881942
+  nsecs: 163871288
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881942.619282]: received mask_obs with header seq: 61
+stamp:
+  secs: 1733881942
+  nsecs: 617688417
+frame_id: '' and grasped_object_mask header seq: 8104
+stamp:
+  secs: 1733881942
+  nsecs: 397009134
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881942.806585]: received mask_obs with header seq: 62
+stamp:
+  secs: 1733881942
+  nsecs: 805705308
+frame_id: '' and grasped_object_mask header seq: 8109
+stamp:
+  secs: 1733881942
+  nsecs: 563535213
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881943.003788]: received mask_obs with header seq: 63
+stamp:
+  secs: 1733881943
+  nsecs:   2786159
+frame_id: '' and grasped_object_mask header seq: 8115
+stamp:
+  secs: 1733881942
+  nsecs: 763370991
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881943.203472]: received mask_obs with header seq: 64
+stamp:
+  secs: 1733881943
+  nsecs: 202393293
+frame_id: '' and grasped_object_mask header seq: 8121
+stamp:
+  secs: 1733881942
+  nsecs: 963209629
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881943.406879]: received mask_obs with header seq: 65
+stamp:
+  secs: 1733881943
+  nsecs: 405942440
+frame_id: '' and grasped_object_mask header seq: 8127
+stamp:
+  secs: 1733881943
+  nsecs: 163049221
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881943.602010]: received mask_obs with header seq: 66
+stamp:
+  secs: 1733881943
+  nsecs: 600990533
+frame_id: '' and grasped_object_mask header seq: 8133
+stamp:
+  secs: 1733881943
+  nsecs: 362887859
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881943.805436]: received mask_obs with header seq: 67
+stamp:
+  secs: 1733881943
+  nsecs: 804544448
+frame_id: '' and grasped_object_mask header seq: 8139
+stamp:
+  secs: 1733881943
+  nsecs: 562726736
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881944.003641]: received mask_obs with header seq: 68
+stamp:
+  secs: 1733881944
+  nsecs:   2423763
+frame_id: '' and grasped_object_mask header seq: 8145
+stamp:
+  secs: 1733881943
+  nsecs: 762568951
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881944.201632]: received mask_obs with header seq: 69
+stamp:
+  secs: 1733881944
+  nsecs: 200727224
+frame_id: '' and grasped_object_mask header seq: 8151
+stamp:
+  secs: 1733881943
+  nsecs: 962414980
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881944.411565]: received mask_obs with header seq: 70
+stamp:
+  secs: 1733881944
+  nsecs: 410659313
+frame_id: '' and grasped_object_mask header seq: 8157
+stamp:
+  secs: 1733881944
+  nsecs: 162261248
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881944.608529]: received mask_obs with header seq: 71
+stamp:
+  secs: 1733881944
+  nsecs: 607629060
+frame_id: '' and grasped_object_mask header seq: 8163
+stamp:
+  secs: 1733881944
+  nsecs: 362109184
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881944.799691]: received mask_obs with header seq: 72
+stamp:
+  secs: 1733881944
+  nsecs: 798696279
+frame_id: '' and grasped_object_mask header seq: 8169
+stamp:
+  secs: 1733881944
+  nsecs: 561956167
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881945.002218]: received mask_obs with header seq: 73
+stamp:
+  secs: 1733881945
+  nsecs:   1061439
+frame_id: '' and grasped_object_mask header seq: 8175
+stamp:
+  secs: 1733881944
+  nsecs: 761801004
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881945.202708]: received mask_obs with header seq: 74
+stamp:
+  secs: 1733881945
+  nsecs: 201566219
+frame_id: '' and grasped_object_mask header seq: 8181
+stamp:
+  secs: 1733881944
+  nsecs: 961643934
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881945.408593]: received mask_obs with header seq: 75
+stamp:
+  secs: 1733881945
+  nsecs: 407604694
+frame_id: '' and grasped_object_mask header seq: 8187
+stamp:
+  secs: 1733881945
+  nsecs: 161487103
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881945.601663]: received mask_obs with header seq: 76
+stamp:
+  secs: 1733881945
+  nsecs: 600501298
+frame_id: '' and grasped_object_mask header seq: 8193
+stamp:
+  secs: 1733881945
+  nsecs: 361330271
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881945.808530]: received mask_obs with header seq: 77
+stamp:
+  secs: 1733881945
+  nsecs: 807325363
+frame_id: '' and grasped_object_mask header seq: 8199
+stamp:
+  secs: 1733881945
+  nsecs: 561174870
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881946.008040]: received mask_obs with header seq: 78
+stamp:
+  secs: 1733881946
+  nsecs:   7133245
+frame_id: '' and grasped_object_mask header seq: 8205
+stamp:
+  secs: 1733881945
+  nsecs: 761015177
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881946.201357]: received mask_obs with header seq: 79
+stamp:
+  secs: 1733881946
+  nsecs: 200472593
+frame_id: '' and grasped_object_mask header seq: 8211
+stamp:
+  secs: 1733881945
+  nsecs: 960851908
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881946.401407]: received mask_obs with header seq: 80
+stamp:
+  secs: 1733881946
+  nsecs: 400375843
+frame_id: '' and grasped_object_mask header seq: 8217
+stamp:
+  secs: 1733881946
+  nsecs: 160688162
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881946.607615]: received mask_obs with header seq: 81
+stamp:
+  secs: 1733881946
+  nsecs: 606504678
+frame_id: '' and grasped_object_mask header seq: 8223
+stamp:
+  secs: 1733881946
+  nsecs: 360524893
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881946.810744]: received mask_obs with header seq: 82
+stamp:
+  secs: 1733881946
+  nsecs: 809859037
+frame_id: '' and grasped_object_mask header seq: 8229
+stamp:
+  secs: 1733881946
+  nsecs: 560361385
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881947.001563]: received mask_obs with header seq: 83
+stamp:
+  secs: 1733881947
+  nsecs:    424861
+frame_id: '' and grasped_object_mask header seq: 8235
+stamp:
+  secs: 1733881946
+  nsecs: 760201216
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881947.203573]: received mask_obs with header seq: 84
+stamp:
+  secs: 1733881947
+  nsecs: 202486991
+frame_id: '' and grasped_object_mask header seq: 8241
+stamp:
+  secs: 1733881946
+  nsecs: 960042477
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881947.405680]: received mask_obs with header seq: 85
+stamp:
+  secs: 1733881947
+  nsecs: 404779672
+frame_id: '' and grasped_object_mask header seq: 8247
+stamp:
+  secs: 1733881947
+  nsecs: 159883976
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881947.599595]: received mask_obs with header seq: 86
+stamp:
+  secs: 1733881947
+  nsecs: 598672866
+frame_id: '' and grasped_object_mask header seq: 8253
+stamp:
+  secs: 1733881947
+  nsecs: 359725475
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881947.800599]: received mask_obs with header seq: 87
+stamp:
+  secs: 1733881947
+  nsecs: 799697399
+frame_id: '' and grasped_object_mask header seq: 8259
+stamp:
+  secs: 1733881947
+  nsecs: 559567451
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881947.999780]: received mask_obs with header seq: 88
+stamp:
+  secs: 1733881947
+  nsecs: 998712778
+frame_id: '' and grasped_object_mask header seq: 8265
+stamp:
+  secs: 1733881947
+  nsecs: 759409428
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881948.207681]: received mask_obs with header seq: 89
+stamp:
+  secs: 1733881948
+  nsecs: 206808567
+frame_id: '' and grasped_object_mask header seq: 8271
+stamp:
+  secs: 1733881947
+  nsecs: 959252834
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881948.404581]: received mask_obs with header seq: 90
+stamp:
+  secs: 1733881948
+  nsecs: 403736352
+frame_id: '' and grasped_object_mask header seq: 8277
+stamp:
+  secs: 1733881948
+  nsecs: 159095764
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881948.607006]: received mask_obs with header seq: 91
+stamp:
+  secs: 1733881948
+  nsecs: 605962514
+frame_id: '' and grasped_object_mask header seq: 8283
+stamp:
+  secs: 1733881948
+  nsecs: 358938456
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881948.826154]: received mask_obs with header seq: 92
+stamp:
+  secs: 1733881948
+  nsecs: 824640035
+frame_id: '' and grasped_object_mask header seq: 8289
+stamp:
+  secs: 1733881948
+  nsecs: 558781624
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881949.042130]: received mask_obs with header seq: 93
+stamp:
+  secs: 1733881949
+  nsecs:  41005611
+frame_id: '' and grasped_object_mask header seq: 8296
+stamp:
+  secs: 1733881948
+  nsecs: 791931868
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881949.253016]: received mask_obs with header seq: 94
+stamp:
+  secs: 1733881949
+  nsecs: 251410484
+frame_id: '' and grasped_object_mask header seq: 8302
+stamp:
+  secs: 1733881948
+  nsecs: 991774082
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881949.451710]: received mask_obs with header seq: 95
+stamp:
+  secs: 1733881949
+  nsecs: 449931383
+frame_id: '' and grasped_object_mask header seq: 8308
+stamp:
+  secs: 1733881949
+  nsecs: 191616774
+frame_id: "camera_color_optical_frame"
+For topic gripper_width: timestamp difference is 110894429 for nearest: 1733881949502353656 - target: 1733881949391459227 at idx 249
+[INFO] [1733881949.644919]: received mask_obs with header seq: 96
+stamp:
+  secs: 1733881949
+  nsecs: 644002914
+frame_id: '' and grasped_object_mask header seq: 8314
+stamp:
+  secs: 1733881949
+  nsecs: 391459227
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881949.839658]: received mask_obs with header seq: 97
+stamp:
+  secs: 1733881949
+  nsecs: 838388442
+frame_id: '' and grasped_object_mask header seq: 8320
+stamp:
+  secs: 1733881949
+  nsecs: 591301680
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881950.026284]: received mask_obs with header seq: 98
+stamp:
+  secs: 1733881950
+  nsecs:  24919033
+frame_id: '' and grasped_object_mask header seq: 8326
+stamp:
+  secs: 1733881949
+  nsecs: 791146994
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881950.241674]: received mask_obs with header seq: 99
+stamp:
+  secs: 1733881950
+  nsecs: 240784645
+frame_id: '' and grasped_object_mask header seq: 8332
+stamp:
+  secs: 1733881949
+  nsecs: 990992785
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881950.441060]: received mask_obs with header seq: 100
+stamp:
+  secs: 1733881950
+  nsecs: 439967393
+frame_id: '' and grasped_object_mask header seq: 8338
+stamp:
+  secs: 1733881950
+  nsecs: 190838814
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881950.630870]: received mask_obs with header seq: 101
+stamp:
+  secs: 1733881950
+  nsecs: 629913806
+frame_id: '' and grasped_object_mask header seq: 8344
+stamp:
+  secs: 1733881950
+  nsecs: 390685320
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881950.832320]: received mask_obs with header seq: 102
+stamp:
+  secs: 1733881950
+  nsecs: 831316471
+frame_id: '' and grasped_object_mask header seq: 8350
+stamp:
+  secs: 1733881950
+  nsecs: 590532303
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881951.028132]: received mask_obs with header seq: 103
+stamp:
+  secs: 1733881951
+  nsecs:  27282714
+frame_id: '' and grasped_object_mask header seq: 8356
+stamp:
+  secs: 1733881950
+  nsecs: 790374041
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881951.215708]: received mask_obs with header seq: 104
+stamp:
+  secs: 1733881951
+  nsecs: 214742898
+frame_id: '' and grasped_object_mask header seq: 8362
+stamp:
+  secs: 1733881950
+  nsecs: 990212679
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881951.419554]: received mask_obs with header seq: 105
+stamp:
+  secs: 1733881951
+  nsecs: 418620824
+frame_id: '' and grasped_object_mask header seq: 8368
+stamp:
+  secs: 1733881951
+  nsecs: 190052032
+frame_id: "camera_color_optical_frame"
+[rosbag_record-3] killing on exit
+[pub_text-1] killing on exit[print_text-2] killing on exit
+[INFO] [1733881951.619548]: received mask_obs with header seq: 106
+stamp:
+  secs: 1733881951
+  nsecs: 603054285
+frame_id: '' and grasped_object_mask header seq: 8373
+stamp:
+  secs: 1733881951
+  nsecs: 356583834
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881951.832582]: received mask_obs with header seq: 107
+stamp:
+  secs: 1733881951
+  nsecs: 831149339
+frame_id: '' and grasped_object_mask header seq: 8380
+stamp:
+  secs: 1733881951
+  nsecs: 589729309
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881952.050082]: received mask_obs with header seq: 108
+stamp:
+  secs: 1733881952
+  nsecs:  46733140
+frame_id: '' and grasped_object_mask header seq: 8386
+stamp:
+  secs: 1733881951
+  nsecs: 789563179
+frame_id: "camera_color_optical_frame"
+[INFO] [1733881952.244104]: received mask_obs with header seq: 109
+stamp:
+  secs: 1733881952
+  nsecs: 243089437
+frame_id: '' and grasped_object_mask header seq: 8393
+stamp:
+  secs: 1733881952
+  nsecs:  22698879
+frame_id: "camera_color_optical_frame"
+Exception in thread Thread-62:
+Traceback (most recent call last):
+  File "/home/leonmkim/.pyenv/versions/3.10.14/lib/python3.10/threading.py", line 1016, in _bootstrap_inner
+    self.run()
+  File "/opt/ros/noetic/lib/python3/dist-packages/rospy/timer.py", line 237, in run
+    self._callback(TimerEvent(last_expected, last_real, current_expected, current_real, last_duration))
+  File "/home/leonmkim/fish_leon/FISH/video.py", line 78, in ros_record_cb
+    self.process.stdin.write(frame.tobytes())
+BrokenPipeError: [Errno 32] Broken pipe
+program exiting gracefully
+Exception in thread Thread-71:
+Traceback (most recent call last):
+  File "/home/leonmkim/.pyenv/versions/3.10.14/lib/python3.10/threading.py", line 1016, in _bootstrap_inner
+    self.run()
+  File "/opt/ros/noetic/lib/python3/dist-packages/rospy/timer.py", line 237, in run
+    self._callback(TimerEvent(last_expected, last_real, current_expected, current_real, last_duration))
+  File "/home/leonmkim/fish_leon/gym-envs/gym_envs/envs/franka_env.py", line 585, in ros_step_cb
+    mask_header, mask_obs_msg = self.wait_for_exact_match(frame_metadata['rgb_header'].stamp, self.mask_obs_queue)
+  File "/home/leonmkim/fish_leon/gym-envs/gym_envs/envs/franka_env.py", line 636, in wait_for_exact_match
+    assert queue_return is not None, f"Exact match not found in {timeout} seconds"
+AssertionError: Exact match not found in 5 seconds
+proceeding to start episode!
+[INFO] [1733881964.331730]: Saving video...
+[INFO] [1733881964.332162]: Video saved!

205123/wandb/run-20241210_205130-svysfc56/files/requirements.txt

@@ -0,0 +1,339 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.2
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0

205123/wandb/run-20241210_205130-svysfc56/files/wandb-metadata.json

@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2024-12-11T01:51:30.761221",
+    "startedAt": "2024-12-11T01:51:30.302896",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "4f7c0421eb84a7caf6bf5670f68777523362676f"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 4124.7015,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.948,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.997,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.999,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 364.1780319213867
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71596908569336
+    }
+}

205123/wandb/run-20241210_205130-svysfc56/files/wandb-summary.json

@@ -0,0 +1 @@
+{"eval/video": {"_type": "video-file", "sha256": "b6b0b3ba17b7513a5df9bca8084b0eda15d2f82df085783449312a47400bba10", "size": 942048, "path": "media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4"}, "global_step": 109, "_timestamp": 1733881964.3609016, "_runtime": 74.04860258102417, "_step": 0, "_wandb": {"runtime": 73}}

205123/wandb/run-20241210_205130-svysfc56/logs/debug-internal.log

@@ -0,0 +1,351 @@
+2024-12-10 20:51:30,311 INFO    StreamThr :597014 [internal.py:wandb_internal():85] W&B internal server running at pid: 597014, started at: 2024-12-10 20:51:30.311157
+2024-12-10 20:51:30,312 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status
+2024-12-10 20:51:30,325 INFO    WriterThread:597014 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/run-svysfc56.wandb
+2024-12-10 20:51:30,325 DEBUG   SenderThread:597014 [sender.py:send():379] send: header
+2024-12-10 20:51:30,336 DEBUG   SenderThread:597014 [sender.py:send():379] send: run
+2024-12-10 20:51:30,645 INFO    SenderThread:597014 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files
+2024-12-10 20:51:30,645 INFO    SenderThread:597014 [sender.py:_start_run_threads():1188] run started: svysfc56 with start time 1733881890.312299
+2024-12-10 20:51:30,656 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: check_version
+2024-12-10 20:51:30,656 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: check_version
+2024-12-10 20:51:30,743 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: run_start
+2024-12-10 20:51:30,748 DEBUG   HandlerThread:597014 [system_info.py:__init__():26] System info init
+2024-12-10 20:51:30,748 DEBUG   HandlerThread:597014 [system_info.py:__init__():41] System info init done
+2024-12-10 20:51:30,748 INFO    HandlerThread:597014 [system_monitor.py:start():194] Starting system monitor
+2024-12-10 20:51:30,748 INFO    SystemMonitor:597014 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-10 20:51:30,749 INFO    HandlerThread:597014 [system_monitor.py:probe():214] Collecting system info
+2024-12-10 20:51:30,749 INFO    SystemMonitor:597014 [interfaces.py:start():188] Started cpu monitoring
+2024-12-10 20:51:30,750 INFO    SystemMonitor:597014 [interfaces.py:start():188] Started disk monitoring
+2024-12-10 20:51:30,750 INFO    SystemMonitor:597014 [interfaces.py:start():188] Started gpu monitoring
+2024-12-10 20:51:30,751 INFO    SystemMonitor:597014 [interfaces.py:start():188] Started memory monitoring
+2024-12-10 20:51:30,751 INFO    SystemMonitor:597014 [interfaces.py:start():188] Started network monitoring
+2024-12-10 20:51:30,761 DEBUG   HandlerThread:597014 [system_info.py:probe():152] Probing system
+2024-12-10 20:51:30,762 DEBUG   HandlerThread:597014 [system_info.py:_probe_git():137] Probing git
+2024-12-10 20:51:30,767 DEBUG   HandlerThread:597014 [system_info.py:_probe_git():145] Probing git done
+2024-12-10 20:51:30,767 DEBUG   HandlerThread:597014 [system_info.py:probe():200] Probing system done
+2024-12-10 20:51:30,767 DEBUG   HandlerThread:597014 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-11T01:51:30.761221', 'startedAt': '2024-12-11T01:51:30.302896', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '4f7c0421eb84a7caf6bf5670f68777523362676f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4124.7015, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.948, 'min': 800.0, 'max': 4700.0}, {'current': 4499.997, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 364.1780319213867}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71596908569336}}
+2024-12-10 20:51:30,768 INFO    HandlerThread:597014 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-10 20:51:30,768 INFO    HandlerThread:597014 [system_monitor.py:probe():227] Publishing system info
+2024-12-10 20:51:30,768 DEBUG   HandlerThread:597014 [system_info.py:_save_code():44] Saving code
+2024-12-10 20:51:30,769 DEBUG   HandlerThread:597014 [system_info.py:_save_code():67] Saving code done
+2024-12-10 20:51:30,769 DEBUG   HandlerThread:597014 [system_info.py:_save_patches():84] Saving git patches
+2024-12-10 20:51:30,832 DEBUG   HandlerThread:597014 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-10 20:51:30,832 INFO    HandlerThread:597014 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-10 20:51:30,836 DEBUG   SenderThread:597014 [sender.py:send():379] send: files
+2024-12-10 20:51:30,836 INFO    SenderThread:597014 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-10 20:51:30,836 INFO    SenderThread:597014 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-10 20:51:30,836 INFO    SenderThread:597014 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-10 20:51:31,124 INFO    wandb-upload_2:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/tiz5ae9w-diff.patch
+2024-12-10 20:51:31,126 INFO    wandb-upload_0:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/p6o167mp-wandb-metadata.json
+2024-12-10 20:51:31,143 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-10 20:51:31,144 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: python_packages
+2024-12-10 20:51:31,145 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-10 20:51:31,145 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: stop_status
+2024-12-10 20:51:31,146 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:31,147 INFO    HandlerThread:597014 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/tb"
+  save: true
+}
+control {
+  relay_id: "42f7013eb6ef4a01b21dd778abc8c686"
+}
+_info {
+  stream_id: "svysfc56"
+}
+
+2024-12-10 20:51:31,147 DEBUG   HandlerThread:597014 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-10 20:51:31,173 INFO    wandb-upload_1:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/9ln43j5e-code/FISH/eval_robot.py
+2024-12-10 20:51:31,218 DEBUG   SenderThread:597014 [sender.py:send():379] send: telemetry
+2024-12-10 20:51:31,218 DEBUG   SenderThread:597014 [sender.py:send():379] send: tbrecord
+2024-12-10 20:51:31,218 DEBUG   SenderThread:597014 [sender.py:send():379] send: files
+2024-12-10 20:51:31,218 INFO    SenderThread:597014 [sender.py:_save_file():1454] saving file events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0 with policy live
+2024-12-10 20:51:31,415 INFO    wandb-upload_2:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/08emxf8k-events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0
+2024-12-10 20:51:31,647 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/diff.patch
+2024-12-10 20:51:31,647 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/code/FISH/eval_robot.py
+2024-12-10 20:51:31,647 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/requirements.txt
+2024-12-10 20:51:31,647 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:51:31,647 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-metadata.json
+2024-12-10 20:51:31,647 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0
+2024-12-10 20:51:31,647 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/code/FISH
+2024-12-10 20:51:31,647 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/code
+2024-12-10 20:51:32,144 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:33,145 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:33,646 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:51:34,145 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:35,145 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:36,144 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:51:36,145 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:37,145 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:38,145 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:39,145 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:40,145 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:41,144 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:51:41,146 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:42,146 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:43,146 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:44,146 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:45,146 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:46,144 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-10 20:51:46,144 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: stop_status
+2024-12-10 20:51:46,186 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:46,300 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:51:46,908 DEBUG   SenderThread:597014 [sender.py:send():379] send: config
+2024-12-10 20:51:47,146 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:48,146 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:49,146 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:49,651 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:51:50,146 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:51,147 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:51,335 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:51:52,147 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:53,147 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:53,652 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:51:54,147 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:55,147 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:56,147 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:57,145 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:51:57,147 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:57,654 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:51:58,148 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:51:59,148 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:00,148 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:01,144 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-10 20:52:01,144 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: stop_status
+2024-12-10 20:52:01,189 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:02,148 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:02,300 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:52:02,655 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/config.yaml
+2024-12-10 20:52:03,148 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:04,149 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:05,149 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:06,182 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:07,149 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:07,657 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:08,149 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:08,194 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:52:09,149 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:09,657 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:10,149 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:11,149 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:11,658 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:12,149 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:13,149 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:13,219 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:52:13,659 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:14,150 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:15,150 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:15,659 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:16,144 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-10 20:52:16,144 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: stop_status
+2024-12-10 20:52:16,150 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:17,150 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:17,660 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:18,150 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:18,404 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:52:19,152 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:19,661 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:20,152 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:21,152 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:21,661 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:22,152 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:23,152 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:23,407 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:52:23,662 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:24,152 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:25,153 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:25,662 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:26,153 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:27,153 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:27,663 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:28,153 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:28,607 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:52:29,153 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:29,664 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:30,154 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:30,752 DEBUG   SystemMonitor:597014 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-10 20:52:30,752 DEBUG   SenderThread:597014 [sender.py:send():379] send: stats
+2024-12-10 20:52:31,144 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-10 20:52:31,144 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: stop_status
+2024-12-10 20:52:31,154 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:31,666 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:32,154 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:33,154 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:33,667 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:33,908 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:52:34,154 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:34,667 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:35,154 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:35,668 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:36,154 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:37,155 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:38,155 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:38,668 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:39,155 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:39,371 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:52:40,155 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:41,155 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:42,155 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:43,155 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:44,155 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:44,374 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-10 20:52:44,375 DEBUG   SenderThread:597014 [sender.py:send():379] send: files
+2024-12-10 20:52:44,375 INFO    SenderThread:597014 [sender.py:_save_file():1454] saving file media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4 with policy now
+2024-12-10 20:52:44,375 DEBUG   SenderThread:597014 [sender.py:send():379] send: history
+2024-12-10 20:52:44,375 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:52:44,375 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: summary_record
+2024-12-10 20:52:44,377 INFO    SenderThread:597014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-10 20:52:44,377 DEBUG   SenderThread:597014 [sender.py:send():379] send: exit
+2024-12-10 20:52:44,377 INFO    SenderThread:597014 [sender.py:send_exit():586] handling exit code: 0
+2024-12-10 20:52:44,377 INFO    SenderThread:597014 [sender.py:send_exit():588] handling runtime: 73
+2024-12-10 20:52:44,377 INFO    SenderThread:597014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-10 20:52:44,377 INFO    SenderThread:597014 [sender.py:send_exit():594] send defer
+2024-12-10 20:52:44,379 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:44,379 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-10 20:52:44,380 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:44,380 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-10 20:52:44,380 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 1
+2024-12-10 20:52:44,380 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:44,380 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-10 20:52:44,380 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:44,380 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-10 20:52:44,380 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 2
+2024-12-10 20:52:44,380 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:44,380 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-10 20:52:44,380 INFO    HandlerThread:597014 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-10 20:52:44,381 DEBUG   SystemMonitor:597014 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-10 20:52:44,381 DEBUG   SystemMonitor:597014 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-10 20:52:44,381 INFO    HandlerThread:597014 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-10 20:52:44,382 INFO    HandlerThread:597014 [interfaces.py:finish():200] Joined disk monitor
+2024-12-10 20:52:44,439 INFO    HandlerThread:597014 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-10 20:52:44,439 INFO    HandlerThread:597014 [interfaces.py:finish():200] Joined memory monitor
+2024-12-10 20:52:44,439 INFO    HandlerThread:597014 [interfaces.py:finish():200] Joined network monitor
+2024-12-10 20:52:44,440 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:44,440 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-10 20:52:44,440 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 3
+2024-12-10 20:52:44,440 DEBUG   SenderThread:597014 [sender.py:send():379] send: stats
+2024-12-10 20:52:44,440 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:44,440 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-10 20:52:44,440 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:44,440 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-10 20:52:44,440 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 4
+2024-12-10 20:52:44,441 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:44,441 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-10 20:52:44,670 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-summary.json
+2024-12-10 20:52:44,671 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4
+2024-12-10 20:52:44,671 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/media/videos/eval
+2024-12-10 20:52:44,671 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/media/videos
+2024-12-10 20:52:44,671 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/media
+2024-12-10 20:52:44,696 INFO    wandb-upload_1:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/6w704qfx-media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4
+2024-12-10 20:52:45,030 INFO    wandb-upload_2:597014 [upload_job.py:push():130] Uploaded file /tmp/tmp01kl0ofywandb/kk0w0e5k-media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4
+2024-12-10 20:52:45,671 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:51,156 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-10 20:52:51,156 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:52:51,156 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-10 20:52:51,157 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:51,157 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-10 20:52:51,157 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 5
+2024-12-10 20:52:51,157 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: poll_exit
+2024-12-10 20:52:51,157 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:51,157 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-10 20:52:51,157 DEBUG   SenderThread:597014 [sender.py:send():379] send: summary
+2024-12-10 20:52:51,158 INFO    SenderThread:597014 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-10 20:52:51,158 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:51,158 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-10 20:52:51,158 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 6
+2024-12-10 20:52:51,158 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:51,158 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-10 20:52:51,158 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:51,158 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-10 20:52:51,158 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 7
+2024-12-10 20:52:51,158 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:52:51,158 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:51,158 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-10 20:52:51,159 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:51,159 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-10 20:52:51,375 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-10 20:52:51,673 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-summary.json
+2024-12-10 20:52:51,921 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 8
+2024-12-10 20:52:51,922 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: poll_exit
+2024-12-10 20:52:51,922 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:51,922 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-10 20:52:51,922 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:51,922 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-10 20:52:51,922 INFO    SenderThread:597014 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-10 20:52:51,924 INFO    SenderThread:597014 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-10 20:52:51,942 INFO    SenderThread:597014 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-10 20:52:51,951 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 9
+2024-12-10 20:52:51,952 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:51,952 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-10 20:52:51,952 DEBUG   SenderThread:597014 [sender.py:send():379] send: artifact
+2024-12-10 20:52:52,376 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-10 20:52:52,626 INFO    wandb-upload_0:597014 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmptoffctew
+2024-12-10 20:52:52,673 INFO    Thread-12 :597014 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:52,858 INFO    wandb-upload_2:597014 [upload_job.py:push():88] Uploaded file /tmp/tmpemx2d4yl/wandb-job.json
+2024-12-10 20:52:52,880 INFO    wandb-upload_1:597014 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpc2y7tffk
+2024-12-10 20:52:53,418 INFO    SenderThread:597014 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM2Njc1OTg1Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM1MTcyNjQ1OA==', 'versionIndex': 335}}}
+2024-12-10 20:52:53,418 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:53,418 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-10 20:52:53,418 INFO    SenderThread:597014 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-10 20:52:53,674 INFO    SenderThread:597014 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files
+2024-12-10 20:52:53,674 INFO    SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/config.yaml config.yaml
+2024-12-10 20:52:53,674 INFO    SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-summary.json wandb-summary.json
+2024-12-10 20:52:53,674 INFO    SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/requirements.txt requirements.txt
+2024-12-10 20:52:53,674 INFO    SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-metadata.json wandb-metadata.json
+2024-12-10 20:52:53,674 INFO    SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0 events.out.tfevents.1733881891.leonmkim-ROG-Strix-G15CS-G15CS.596892.0
+2024-12-10 20:52:53,674 INFO    SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log output.log
+2024-12-10 20:52:53,676 INFO    SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/diff.patch diff.patch
+2024-12-10 20:52:53,680 INFO    SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-10 20:52:53,680 INFO    SenderThread:597014 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4 media/videos/eval/video_0_b6b0b3ba17b7513a5df9.mp4
+2024-12-10 20:52:53,680 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 10
+2024-12-10 20:52:53,680 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: poll_exit
+2024-12-10 20:52:53,681 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:53,681 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-10 20:52:53,681 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:53,681 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-10 20:52:53,681 INFO    SenderThread:597014 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-10 20:52:53,872 INFO    wandb-upload_2:597014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/requirements.txt
+2024-12-10 20:52:53,888 INFO    wandb-upload_0:597014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/config.yaml
+2024-12-10 20:52:53,899 INFO    wandb-upload_3:597014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/wandb-summary.json
+2024-12-10 20:52:53,910 INFO    wandb-upload_1:597014 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/files/output.log
+2024-12-10 20:52:54,110 INFO    Thread-11 (_thread_body):597014 [sender.py:transition_state():614] send defer: 11
+2024-12-10 20:52:54,110 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:54,111 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-10 20:52:54,111 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:54,111 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-10 20:52:54,111 INFO    SenderThread:597014 [file_pusher.py:join():175] waiting for file pusher
+2024-12-10 20:52:54,111 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 12
+2024-12-10 20:52:54,111 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:54,111 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-10 20:52:54,112 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:54,112 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-10 20:52:54,112 INFO    SenderThread:597014 [file_stream.py:finish():601] file stream finish called
+2024-12-10 20:52:54,317 INFO    SenderThread:597014 [file_stream.py:finish():605] file stream finish is done
+2024-12-10 20:52:54,317 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 13
+2024-12-10 20:52:54,318 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:54,318 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-10 20:52:54,318 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:54,318 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-10 20:52:54,318 INFO    SenderThread:597014 [sender.py:transition_state():614] send defer: 14
+2024-12-10 20:52:54,318 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:52:54,318 INFO    HandlerThread:597014 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-10 20:52:54,319 DEBUG   SenderThread:597014 [sender.py:send():379] send: final
+2024-12-10 20:52:54,319 DEBUG   SenderThread:597014 [sender.py:send():379] send: footer
+2024-12-10 20:52:54,319 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:52:54,319 INFO    SenderThread:597014 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-10 20:52:54,319 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-10 20:52:54,320 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: poll_exit
+2024-12-10 20:52:54,320 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: server_info
+2024-12-10 20:52:54,320 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: server_info
+2024-12-10 20:52:54,322 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-10 20:52:54,322 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-10 20:52:54,322 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-10 20:52:54,322 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:52:54,378 DEBUG   SenderThread:597014 [sender.py:send_request():406] send_request: poll_exit
+2024-12-10 20:52:54,378 INFO    MainThread:597014 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-10 20:52:54,378 INFO    MainThread:597014 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-10 20:52:54,379 INFO    MainThread:597014 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-10 20:52:54,379 DEBUG   HandlerThread:597014 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-10 20:52:54,379 INFO    HandlerThread:597014 [handler.py:finish():882] shutting down handler
+2024-12-10 20:52:55,322 INFO    WriterThread:597014 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/run-svysfc56.wandb
+2024-12-10 20:52:55,378 INFO    SenderThread:597014 [sender.py:finish():1615] shutting down sender
+2024-12-10 20:52:55,378 INFO    SenderThread:597014 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-10 20:52:55,378 INFO    SenderThread:597014 [file_pusher.py:join():175] waiting for file pusher

205123/wandb/run-20241210_205130-svysfc56/logs/debug.log

@@ -0,0 +1,31 @@
+2024-12-10 20:51:30,303 INFO    MainThread:596892 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-10 20:51:30,303 INFO    MainThread:596892 [wandb_setup.py:_flush():76] Configure stats pid to 596892
+2024-12-10 20:51:30,303 INFO    MainThread:596892 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-10 20:51:30,303 INFO    MainThread:596892 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/settings
+2024-12-10 20:51:30,303 INFO    MainThread:596892 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-10 20:51:30,303 INFO    MainThread:596892 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-10 20:51:30,304 INFO    MainThread:596892 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-10 20:51:30,304 INFO    MainThread:596892 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-10 20:51:30,304 INFO    MainThread:596892 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/logs/debug.log
+2024-12-10 20:51:30,304 INFO    MainThread:596892 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/wandb/run-20241210_205130-svysfc56/logs/debug-internal.log
+2024-12-10 20:51:30,304 INFO    MainThread:596892 [wandb_init.py:init():569] calling init triggers
+2024-12-10 20:51:30,304 INFO    MainThread:596892 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '265626', 'wandb_notes': '265626_req_265827', 'eval': True, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim_left_to_right'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'obs_shape': [5, 240, 320], 'action_shape': [36, 7], 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}, 'true_action_history': True}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'true_action_history': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt'}
+2024-12-10 20:51:30,304 INFO    MainThread:596892 [wandb_init.py:init():619] starting backend
+2024-12-10 20:51:30,304 INFO    MainThread:596892 [wandb_init.py:init():623] setting up manager
+2024-12-10 20:51:30,310 INFO    MainThread:596892 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-10 20:51:30,312 INFO    MainThread:596892 [wandb_init.py:init():631] backend started and connected
+2024-12-10 20:51:30,324 INFO    MainThread:596892 [wandb_init.py:init():720] updated telemetry
+2024-12-10 20:51:30,334 INFO    MainThread:596892 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-10 20:51:30,656 INFO    MainThread:596892 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-10 20:51:30,737 INFO    MainThread:596892 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.0 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-10 20:51:30,737 INFO    MainThread:596892 [wandb_init.py:init():804] starting run threads in backend
+2024-12-10 20:51:31,144 INFO    MainThread:596892 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-10 20:51:31,145 INFO    MainThread:596892 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-10 20:51:31,145 INFO    MainThread:596892 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-10 20:51:31,145 INFO    MainThread:596892 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-10 20:51:31,147 INFO    MainThread:596892 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-10 20:51:31,147 INFO    MainThread:596892 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205123/tb, True
+2024-12-10 20:51:46,908 INFO    MainThread:596892 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'}
+2024-12-10 20:52:55,379 WARNING MsgRouterThr:596892 [router.py:message_loop():77] message_loop has been closed

205307/.hydra/config.yaml

@@ -0,0 +1,349 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '265626'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

205307/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${experiment_dir}/${now:%H%M%S}
+  sweep:
+    dir: ${experiment_dir}/${now:%H%M%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${experiment_dir}/${now:%H%M%S}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_policy
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205307
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

205307/eval_policy.log

@@ -0,0 +1,15 @@
+[2024-12-10 20:53:07,380][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-10 20:53:07,383][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2024-12-10 20:53:08,679][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+

205324/.hydra/config.yaml

@@ -0,0 +1,349 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '265626'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

205324/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${experiment_dir}/${now:%H%M%S}
+  sweep:
+    dir: ${experiment_dir}/${now:%H%M%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${experiment_dir}/${now:%H%M%S}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

205324/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion

205324/episode_rosbags/aligned_depth_to_color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

205324/episode_rosbags/cam_tf_world.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256

205324/episode_rosbags/color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

205324/episode_rosbags/depth_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200

205324/eval_robot.log

@@ -0,0 +1,12 @@
+[2024-12-10 20:53:24,743][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-10 20:53:24,746][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+

205324/wandb/debug-internal.log

@@ -0,0 +1,327 @@
+2024-12-10 20:53:30,850 INFO    StreamThr :598006 [internal.py:wandb_internal():85] W&B internal server running at pid: 598006, started at: 2024-12-10 20:53:30.850039
+2024-12-10 20:53:30,851 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status
+2024-12-10 20:53:30,863 INFO    WriterThread:598006 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/run-52nddla7.wandb
+2024-12-10 20:53:30,863 DEBUG   SenderThread:598006 [sender.py:send():379] send: header
+2024-12-10 20:53:30,875 DEBUG   SenderThread:598006 [sender.py:send():379] send: run
+2024-12-10 20:53:31,147 INFO    SenderThread:598006 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files
+2024-12-10 20:53:31,147 INFO    SenderThread:598006 [sender.py:_start_run_threads():1188] run started: 52nddla7 with start time 1733882010.850819
+2024-12-10 20:53:31,161 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: check_version
+2024-12-10 20:53:31,161 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: check_version
+2024-12-10 20:53:31,206 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: run_start
+2024-12-10 20:53:31,211 DEBUG   HandlerThread:598006 [system_info.py:__init__():26] System info init
+2024-12-10 20:53:31,211 DEBUG   HandlerThread:598006 [system_info.py:__init__():41] System info init done
+2024-12-10 20:53:31,211 INFO    HandlerThread:598006 [system_monitor.py:start():194] Starting system monitor
+2024-12-10 20:53:31,211 INFO    SystemMonitor:598006 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-10 20:53:31,212 INFO    HandlerThread:598006 [system_monitor.py:probe():214] Collecting system info
+2024-12-10 20:53:31,212 INFO    SystemMonitor:598006 [interfaces.py:start():188] Started cpu monitoring
+2024-12-10 20:53:31,212 INFO    SystemMonitor:598006 [interfaces.py:start():188] Started disk monitoring
+2024-12-10 20:53:31,213 INFO    SystemMonitor:598006 [interfaces.py:start():188] Started gpu monitoring
+2024-12-10 20:53:31,213 INFO    SystemMonitor:598006 [interfaces.py:start():188] Started memory monitoring
+2024-12-10 20:53:31,214 INFO    SystemMonitor:598006 [interfaces.py:start():188] Started network monitoring
+2024-12-10 20:53:31,224 DEBUG   HandlerThread:598006 [system_info.py:probe():152] Probing system
+2024-12-10 20:53:31,226 DEBUG   HandlerThread:598006 [system_info.py:_probe_git():137] Probing git
+2024-12-10 20:53:31,231 DEBUG   HandlerThread:598006 [system_info.py:_probe_git():145] Probing git done
+2024-12-10 20:53:31,231 DEBUG   HandlerThread:598006 [system_info.py:probe():200] Probing system done
+2024-12-10 20:53:31,231 DEBUG   HandlerThread:598006 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-11T01:53:31.224621', 'startedAt': '2024-12-11T01:53:30.842354', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '4f7c0421eb84a7caf6bf5670f68777523362676f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4312.5025, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4500.019, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.007, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.943, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 365.2335090637207}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71596908569336}}
+2024-12-10 20:53:31,231 INFO    HandlerThread:598006 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-10 20:53:31,231 INFO    HandlerThread:598006 [system_monitor.py:probe():227] Publishing system info
+2024-12-10 20:53:31,231 DEBUG   HandlerThread:598006 [system_info.py:_save_code():44] Saving code
+2024-12-10 20:53:31,233 DEBUG   HandlerThread:598006 [system_info.py:_save_code():67] Saving code done
+2024-12-10 20:53:31,233 DEBUG   HandlerThread:598006 [system_info.py:_save_patches():84] Saving git patches
+2024-12-10 20:53:31,305 DEBUG   HandlerThread:598006 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-10 20:53:31,306 INFO    HandlerThread:598006 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-10 20:53:31,309 DEBUG   SenderThread:598006 [sender.py:send():379] send: files
+2024-12-10 20:53:31,310 INFO    SenderThread:598006 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-10 20:53:31,310 INFO    SenderThread:598006 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-10 20:53:31,310 INFO    SenderThread:598006 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-10 20:53:31,570 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-10 20:53:31,570 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: python_packages
+2024-12-10 20:53:31,570 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-10 20:53:31,571 INFO    wandb-upload_0:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/oqlixunj-wandb-metadata.json
+2024-12-10 20:53:31,571 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:31,572 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: stop_status
+2024-12-10 20:53:31,573 INFO    HandlerThread:598006 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/tb"
+  save: true
+}
+control {
+  relay_id: "5d813b8bb5a547c8896a7e03615dca84"
+}
+_info {
+  stream_id: "52nddla7"
+}
+
+2024-12-10 20:53:31,574 DEBUG   HandlerThread:598006 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-10 20:53:31,605 INFO    wandb-upload_1:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/6yv6eek3-code/FISH/eval_robot.py
+2024-12-10 20:53:31,648 INFO    wandb-upload_2:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/w33l6u66-diff.patch
+2024-12-10 20:53:31,702 DEBUG   SenderThread:598006 [sender.py:send():379] send: telemetry
+2024-12-10 20:53:31,702 DEBUG   SenderThread:598006 [sender.py:send():379] send: tbrecord
+2024-12-10 20:53:31,703 DEBUG   SenderThread:598006 [sender.py:send():379] send: files
+2024-12-10 20:53:31,703 INFO    SenderThread:598006 [sender.py:_save_file():1454] saving file events.out.tfevents.1733882011.leonmkim-ROG-Strix-G15CS-G15CS.597884.0 with policy live
+2024-12-10 20:53:31,907 INFO    wandb-upload_0:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/uat8jiac-events.out.tfevents.1733882011.leonmkim-ROG-Strix-G15CS-G15CS.597884.0
+2024-12-10 20:53:32,149 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/events.out.tfevents.1733882011.leonmkim-ROG-Strix-G15CS-G15CS.597884.0
+2024-12-10 20:53:32,149 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/code/FISH/eval_robot.py
+2024-12-10 20:53:32,149 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-metadata.json
+2024-12-10 20:53:32,149 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:53:32,149 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/requirements.txt
+2024-12-10 20:53:32,149 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/diff.patch
+2024-12-10 20:53:32,149 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/code/FISH
+2024-12-10 20:53:32,149 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/code
+2024-12-10 20:53:32,571 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:33,571 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:34,148 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:53:34,571 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:35,547 DEBUG   SenderThread:598006 [sender.py:send():379] send: config
+2024-12-10 20:53:35,590 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:36,571 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:53:36,572 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:37,572 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:38,149 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:53:38,572 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:39,572 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:40,150 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:53:40,572 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:41,572 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:42,082 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:53:42,150 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:53:42,572 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:43,572 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:44,151 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:53:44,573 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:45,573 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:46,151 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:53:46,570 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-10 20:53:46,570 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: stop_status
+2024-12-10 20:53:46,614 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:47,573 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:47,640 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:53:48,573 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:49,573 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:50,573 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:51,574 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:52,574 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:53,571 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:53:53,574 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:54,574 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:55,574 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:56,574 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:57,574 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:58,571 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:53:58,574 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:59,575 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:00,575 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:01,570 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-10 20:54:01,570 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: stop_status
+2024-12-10 20:54:01,613 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:02,575 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:03,575 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:03,722 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:54:04,156 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/config.yaml
+2024-12-10 20:54:04,575 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:05,575 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:06,575 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:07,576 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:08,576 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:09,550 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:54:09,576 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:10,158 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:10,576 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:11,576 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:12,159 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:12,577 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:13,577 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:14,159 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:14,567 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:54:14,577 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:15,160 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:15,577 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:16,570 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-10 20:54:16,570 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: stop_status
+2024-12-10 20:54:16,577 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:17,160 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:17,577 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:18,579 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:19,161 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:19,567 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:54:19,577 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:20,577 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:21,161 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:21,577 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:22,578 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:23,162 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:23,578 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:24,578 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:24,607 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:54:25,162 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:25,578 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:26,578 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:27,163 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:27,579 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:28,579 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:29,164 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:29,254 DEBUG   SenderThread:598006 [sender.py:send():379] send: files
+2024-12-10 20:54:29,254 INFO    SenderThread:598006 [sender.py:_save_file():1454] saving file media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4 with policy now
+2024-12-10 20:54:29,256 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-10 20:54:29,257 DEBUG   SenderThread:598006 [sender.py:send():379] send: history
+2024-12-10 20:54:29,257 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: summary_record
+2024-12-10 20:54:29,257 INFO    SenderThread:598006 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-10 20:54:29,258 DEBUG   SenderThread:598006 [sender.py:send():379] send: exit
+2024-12-10 20:54:29,258 INFO    SenderThread:598006 [sender.py:send_exit():586] handling exit code: 0
+2024-12-10 20:54:29,259 INFO    SenderThread:598006 [sender.py:send_exit():588] handling runtime: 58
+2024-12-10 20:54:29,259 INFO    SenderThread:598006 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-10 20:54:29,259 INFO    SenderThread:598006 [sender.py:send_exit():594] send defer
+2024-12-10 20:54:29,259 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:29,259 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-10 20:54:29,259 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:29,259 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-10 20:54:29,259 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 1
+2024-12-10 20:54:29,259 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:29,259 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-10 20:54:29,260 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:29,260 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-10 20:54:29,260 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 2
+2024-12-10 20:54:29,260 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:29,260 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-10 20:54:29,260 INFO    HandlerThread:598006 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-10 20:54:29,260 DEBUG   SystemMonitor:598006 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-10 20:54:29,260 DEBUG   SystemMonitor:598006 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-10 20:54:29,260 DEBUG   SystemMonitor:598006 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-10 20:54:29,269 INFO    HandlerThread:598006 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-10 20:54:29,269 INFO    HandlerThread:598006 [interfaces.py:finish():200] Joined disk monitor
+2024-12-10 20:54:29,319 INFO    HandlerThread:598006 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-10 20:54:29,319 INFO    HandlerThread:598006 [interfaces.py:finish():200] Joined memory monitor
+2024-12-10 20:54:29,320 INFO    HandlerThread:598006 [interfaces.py:finish():200] Joined network monitor
+2024-12-10 20:54:29,320 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:29,320 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-10 20:54:29,320 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 3
+2024-12-10 20:54:29,320 DEBUG   SenderThread:598006 [sender.py:send():379] send: stats
+2024-12-10 20:54:29,320 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:29,320 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-10 20:54:29,321 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:29,321 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-10 20:54:29,321 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 4
+2024-12-10 20:54:29,321 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:29,321 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-10 20:54:29,518 INFO    wandb-upload_2:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/0kw42edo-media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4
+2024-12-10 20:54:30,164 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:30,164 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4
+2024-12-10 20:54:30,164 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-summary.json
+2024-12-10 20:54:30,165 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/media/videos
+2024-12-10 20:54:30,165 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/media/videos/eval
+2024-12-10 20:54:30,165 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/media
+2024-12-10 20:54:30,523 INFO    wandb-upload_0:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/ks8s91l3-media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4
+2024-12-10 20:54:35,581 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-10 20:54:35,581 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:54:35,581 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-10 20:54:35,581 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:54:35,581 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:35,581 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-10 20:54:35,581 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 5
+2024-12-10 20:54:35,581 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit
+2024-12-10 20:54:35,582 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:35,582 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-10 20:54:35,582 DEBUG   SenderThread:598006 [sender.py:send():379] send: summary
+2024-12-10 20:54:35,582 INFO    SenderThread:598006 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-10 20:54:35,582 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:35,582 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-10 20:54:35,582 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 6
+2024-12-10 20:54:35,583 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:35,583 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-10 20:54:35,583 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:35,583 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-10 20:54:35,583 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 7
+2024-12-10 20:54:35,583 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:54:35,583 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:35,583 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-10 20:54:35,583 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:35,583 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-10 20:54:36,166 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-summary.json
+2024-12-10 20:54:36,260 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-10 20:54:37,110 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 8
+2024-12-10 20:54:37,110 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit
+2024-12-10 20:54:37,110 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:37,110 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-10 20:54:37,111 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:37,111 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-10 20:54:37,111 INFO    SenderThread:598006 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-10 20:54:37,113 INFO    SenderThread:598006 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-10 20:54:37,127 INFO    SenderThread:598006 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-10 20:54:37,136 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 9
+2024-12-10 20:54:37,136 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:37,136 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-10 20:54:37,137 DEBUG   SenderThread:598006 [sender.py:send():379] send: artifact
+2024-12-10 20:54:37,166 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:37,260 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-10 20:54:37,780 INFO    wandb-upload_1:598006 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_p_p260o
+2024-12-10 20:54:37,783 INFO    wandb-upload_0:598006 [upload_job.py:push():85] Skipped uploading /tmp/tmppldao70p/wandb-job.json
+2024-12-10 20:54:37,954 INFO    wandb-upload_2:598006 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4q7vuxho
+2024-12-10 20:54:38,567 INFO    SenderThread:598006 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM2Njc2MTczOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM2Njc1OTg1Ng==', 'versionIndex': 336}}}
+2024-12-10 20:54:38,567 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:38,567 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-10 20:54:38,567 INFO    SenderThread:598006 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-10 20:54:39,167 INFO    SenderThread:598006 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files
+2024-12-10 20:54:39,168 INFO    SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/config.yaml config.yaml
+2024-12-10 20:54:39,168 INFO    SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-summary.json wandb-summary.json
+2024-12-10 20:54:39,168 INFO    SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/events.out.tfevents.1733882011.leonmkim-ROG-Strix-G15CS-G15CS.597884.0 events.out.tfevents.1733882011.leonmkim-ROG-Strix-G15CS-G15CS.597884.0
+2024-12-10 20:54:39,168 INFO    SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/requirements.txt requirements.txt
+2024-12-10 20:54:39,168 INFO    SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-metadata.json wandb-metadata.json
+2024-12-10 20:54:39,168 INFO    SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log output.log
+2024-12-10 20:54:39,173 INFO    SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/diff.patch diff.patch
+2024-12-10 20:54:39,173 INFO    SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-10 20:54:39,174 INFO    SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4 media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4
+2024-12-10 20:54:39,175 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 10
+2024-12-10 20:54:39,175 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit
+2024-12-10 20:54:39,175 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:39,175 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-10 20:54:39,175 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:39,175 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-10 20:54:39,175 INFO    SenderThread:598006 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-10 20:54:39,261 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-10 20:54:39,261 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit
+2024-12-10 20:54:39,363 INFO    wandb-upload_0:598006 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-summary.json
+2024-12-10 20:54:39,397 INFO    wandb-upload_2:598006 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/requirements.txt
+2024-12-10 20:54:39,416 INFO    wandb-upload_1:598006 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/config.yaml
+2024-12-10 20:54:39,447 INFO    wandb-upload_3:598006 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:39,647 INFO    Thread-11 (_thread_body):598006 [sender.py:transition_state():614] send defer: 11
+2024-12-10 20:54:39,647 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:39,647 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-10 20:54:39,648 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:39,648 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-10 20:54:39,648 INFO    SenderThread:598006 [file_pusher.py:join():175] waiting for file pusher
+2024-12-10 20:54:39,648 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 12
+2024-12-10 20:54:39,648 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:39,648 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-10 20:54:39,648 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:39,649 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-10 20:54:39,649 INFO    SenderThread:598006 [file_stream.py:finish():601] file stream finish called
+2024-12-10 20:54:39,709 INFO    SenderThread:598006 [file_stream.py:finish():605] file stream finish is done
+2024-12-10 20:54:39,709 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 13
+2024-12-10 20:54:39,709 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:39,709 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-10 20:54:39,709 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:39,709 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-10 20:54:39,709 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 14
+2024-12-10 20:54:39,709 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:39,709 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-10 20:54:39,710 DEBUG   SenderThread:598006 [sender.py:send():379] send: final
+2024-12-10 20:54:39,710 DEBUG   SenderThread:598006 [sender.py:send():379] send: footer
+2024-12-10 20:54:39,710 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:39,710 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-10 20:54:39,710 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-10 20:54:39,710 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: server_info
+2024-12-10 20:54:39,710 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-10 20:54:39,711 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit
+2024-12-10 20:54:39,711 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: server_info
+2024-12-10 20:54:39,712 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-10 20:54:39,713 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-10 20:54:39,713 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:39,793 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit
+2024-12-10 20:54:39,794 INFO    MainThread:598006 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-10 20:54:39,794 INFO    MainThread:598006 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-10 20:54:39,794 INFO    MainThread:598006 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-10 20:54:39,794 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-10 20:54:39,794 INFO    HandlerThread:598006 [handler.py:finish():882] shutting down handler
+2024-12-10 20:54:40,712 INFO    WriterThread:598006 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/run-52nddla7.wandb
+2024-12-10 20:54:40,794 INFO    SenderThread:598006 [sender.py:finish():1615] shutting down sender
+2024-12-10 20:54:40,794 INFO    SenderThread:598006 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-10 20:54:40,794 INFO    SenderThread:598006 [file_pusher.py:join():175] waiting for file pusher

205324/wandb/debug.log

@@ -0,0 +1,31 @@
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_setup.py:_flush():76] Configure stats pid to 597884
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/settings
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/logs/debug.log
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/logs/debug-internal.log
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_init.py:init():569] calling init triggers
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '265626', 'wandb_notes': '265626_req_265827', 'eval': True, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim_left_to_right'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'obs_shape': [5, 240, 320], 'action_shape': [36, 7], 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'true_action_history': False, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt'}
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_init.py:init():619] starting backend
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_init.py:init():623] setting up manager
+2024-12-10 20:53:30,849 INFO    MainThread:597884 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-10 20:53:30,850 INFO    MainThread:597884 [wandb_init.py:init():631] backend started and connected
+2024-12-10 20:53:30,862 INFO    MainThread:597884 [wandb_init.py:init():720] updated telemetry
+2024-12-10 20:53:30,871 INFO    MainThread:597884 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-10 20:53:31,160 INFO    MainThread:597884 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-10 20:53:31,200 INFO    MainThread:597884 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.0 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-10 20:53:31,201 INFO    MainThread:597884 [wandb_init.py:init():804] starting run threads in backend
+2024-12-10 20:53:31,571 INFO    MainThread:597884 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-10 20:53:31,571 INFO    MainThread:597884 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-10 20:53:31,571 INFO    MainThread:597884 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-10 20:53:31,571 INFO    MainThread:597884 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-10 20:53:31,573 INFO    MainThread:597884 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-10 20:53:31,573 INFO    MainThread:597884 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/tb, True
+2024-12-10 20:53:35,546 INFO    MainThread:597884 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'}
+2024-12-10 20:54:40,795 WARNING MsgRouterThr:597884 [router.py:message_loop():77] message_loop has been closed

205324/wandb/run-20241210_205330-52nddla7/files/code/FISH/eval_robot.py

@@ -0,0 +1,512 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)		
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()

205324/wandb/run-20241210_205330-52nddla7/files/config.yaml

@@ -0,0 +1,966 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 0
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '265626'
+wandb_notes:
+  desc: null
+  value: 265626_req_265827
+eval:
+  desc: null
+  value: true
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: true
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: false
+use_sdf_maps:
+  desc: null
+  value: false
+use_normals_maps:
+  desc: null
+  value: false
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value:
+  - hbm_twodim_left_to_right
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 0
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/265626
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    obs_shape:
+    - 5
+    - 240
+    - 320
+    action_shape:
+    - 36
+    - 7
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: false
+        use_sdf_maps: false
+        use_normals_maps: false
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+      policy_frequency: 5
+      interpolation_frequency: 25
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 0
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: true
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: false
+        use_sdf_maps: false
+        use_normals_maps: false
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: true
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: false
+          use_sdf_maps: false
+          use_normals_maps: false
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+        policy_frequency: 5
+        interpolation_frequency: 25
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1733882010
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 16
+      - 23
+      - 35
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: hbm
+left_book_slot:
+  desc: null
+  value: twodim

205324/wandb/run-20241210_205330-52nddla7/files/diff.patch

@@ -0,0 +1,34 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index a9568e3..aa52de5 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -123,6 +123,11 @@ bc_weight_type: 'qfilter' # linear, qfilter
+ 
+ # Load weights
+ load_checkpoint: ${agent.load_checkpoint}
++# 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
++# full dataset
++wandb_run_id: "265626" # RGBD+Mask+act history
++# wandb_run_id: "265628" # RGBD+Mask+contact+act history
++
+ # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # full dataset
+ # 6 step action history
+@@ -135,7 +140,7 @@ load_checkpoint: ${agent.load_checkpoint}
+ # wandb_run_id: '264772_0' # RGBD+act history
+ # wandb_run_id: '264775_0' # RGBD+mask+act history
+ # wandb_run_id: '264776_0' # RGBD+contact+act history
+-wandb_run_id: '264777_0' # RGBD+mask+contact+act history
++# wandb_run_id: '264777_0' # RGBD+mask+contact+act history
+ 
+ # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # 16 held out demos
+@@ -151,7 +156,7 @@ wandb_run_id: '264777_0' # RGBD+mask+contact+act history
+ # wandb_run_id: '250380_0' # RGBD+mask+contact+act history
+ # wandb_run_id: '250370_0' # RGBD+contact+act history
+ 
+-true_action_history: true
++true_action_history: false
+ wandb_notes: null
+ 
+ checkpoint_epoch: 12000 # for depth + all contact

205324/wandb/run-20241210_205330-52nddla7/files/output.log

@@ -0,0 +1,461 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1733882015.547602]: resetting environment
+[INFO] [1733882015.555685]: cleared current plan
+[INFO] [1733882015.557298]: moving to home
+[INFO] [1733882018.360980]: reached home
+[INFO] [1733882018.362525]: reset action history
+[INFO] [1733882021.071264]: environment reset
+Starting episode 0
+[INFO] [1733882021.073619]: resetting environment
+[INFO] [1733882021.078303]: cleared current plan
+[INFO] [1733882021.079103]: moving to home
+[INFO] [1733882022.081897]: reached home
+[INFO] [1733882022.083956]: reset action history
+[INFO] [1733882024.791365]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1733882047.498704]: resetting environment
+[INFO] [1733882047.505692]: cleared current plan
+[INFO] [1733882047.506996]: moving to home
+[INFO] [1733882048.509273]: reached home
+[INFO] [1733882048.510630]: reset action history
+[INFO] [1733882051.218254]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/episode_rosbags/episode_0_2024-12-10-20-54-11.bag
+... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-597884.log
+started roslaunch server http://158.130.50.37:45489/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [598110]
+process[print_text-2]: started with pid [598134]
+process[rosbag_record-3]: started with pid [598135]
+started bagging!
+[INFO] [1733882051.973240]: received mask_obs with header seq: 110
+stamp:
+  secs: 1733882051
+  nsecs: 972197771
+frame_id: '' and grasped_object_mask header seq: 11387
+stamp:
+  secs: 1733882051
+  nsecs: 753522158
+frame_id: "camera_color_optical_frame"
+[INFO] [1733882052.158201]: received mask_obs with header seq: 111
+stamp:
+  secs: 1733882052
+  nsecs: 157648563
+frame_id: '' and grasped_object_mask header seq: 11393
+stamp:
+  secs: 1733882051
+  nsecs: 953104734
+frame_id: "camera_color_optical_frame"
+Exception in thread Thread-71:
+Traceback (most recent call last):
+  File "/home/leonmkim/.pyenv/versions/3.10.14/lib/python3.10/threading.py", line 1016, in _bootstrap_inner
+    self.run()
+  File "/opt/ros/noetic/lib/python3/dist-packages/rospy/timer.py", line 237, in run
+    self._callback(TimerEvent(last_expected, last_real, current_expected, current_real, last_duration))
+  File "/home/leonmkim/fish_leon/gym-envs/gym_envs/envs/franka_env.py", line 562, in ros_step_cb
+    action_history, plan_headers_list = self.arm.get_action_history(
+  File "/home/leonmkim/fish_leon/gym-envs/gym_envs/envs/hardware/franka.py", line 574, in get_action_history
+    action_history, plan_headers_list = self.get_hallucinated_action_history(prev_plan_timestamp, current_timestamp, history_length)
+  File "/home/leonmkim/fish_leon/gym-envs/gym_envs/envs/hardware/franka.py", line 599, in get_hallucinated_action_history
+    assert action_plan_return is not None, f'No plan found after {timeout.to_sec()} seconds'
+AssertionError: No plan found after 1.0 seconds
+[WARN] [1733882053.167214]: Plan exhausted
+[WARN] [1733882053.207056]: Plan exhausted
+[WARN] [1733882053.247010]: Plan exhausted
+[WARN] [1733882053.287040]: Plan exhausted
+[WARN] [1733882053.327405]: Plan exhausted
+[WARN] [1733882053.367146]: Plan exhausted
+[WARN] [1733882053.407281]: Plan exhausted
+[WARN] [1733882053.447035]: Plan exhausted
+[WARN] [1733882053.487113]: Plan exhausted
+[WARN] [1733882053.527055]: Plan exhausted
+[WARN] [1733882053.567043]: Plan exhausted
+[WARN] [1733882053.607055]: Plan exhausted
+[WARN] [1733882053.647050]: Plan exhausted
+[WARN] [1733882053.687051]: Plan exhausted
+[WARN] [1733882053.727048]: Plan exhausted
+[WARN] [1733882053.767034]: Plan exhausted
+[WARN] [1733882053.807052]: Plan exhausted
+[WARN] [1733882053.847048]: Plan exhausted
+[WARN] [1733882053.887264]: Plan exhausted
+[WARN] [1733882053.927126]: Plan exhausted
+[WARN] [1733882053.967045]: Plan exhausted
+[WARN] [1733882054.007052]: Plan exhausted
+[WARN] [1733882054.047050]: Plan exhausted
+[WARN] [1733882054.087042]: Plan exhausted
+[WARN] [1733882054.127052]: Plan exhausted
+[WARN] [1733882054.167037]: Plan exhausted
+[WARN] [1733882054.207051]: Plan exhausted
+[WARN] [1733882054.247047]: Plan exhausted
+[WARN] [1733882054.287087]: Plan exhausted
+[WARN] [1733882054.327114]: Plan exhausted
+[WARN] [1733882054.367050]: Plan exhausted
+[WARN] [1733882054.407066]: Plan exhausted
+[WARN] [1733882054.447061]: Plan exhausted
+[WARN] [1733882054.487144]: Plan exhausted
+[WARN] [1733882054.527057]: Plan exhausted
+[WARN] [1733882054.567049]: Plan exhausted
+[WARN] [1733882054.607058]: Plan exhausted
+[WARN] [1733882054.647049]: Plan exhausted
+[WARN] [1733882054.687141]: Plan exhausted
+[WARN] [1733882054.727052]: Plan exhausted
+[WARN] [1733882054.767101]: Plan exhausted
+[WARN] [1733882054.807044]: Plan exhausted
+[WARN] [1733882054.847019]: Plan exhausted
+[WARN] [1733882054.887033]: Plan exhausted
+[WARN] [1733882054.927117]: Plan exhausted
+[WARN] [1733882054.967021]: Plan exhausted
+[WARN] [1733882055.007079]: Plan exhausted
+[WARN] [1733882055.047052]: Plan exhausted
+[WARN] [1733882055.087068]: Plan exhausted
+[WARN] [1733882055.127060]: Plan exhausted
+[WARN] [1733882055.167279]: Plan exhausted
+[WARN] [1733882055.207070]: Plan exhausted
+[WARN] [1733882055.247038]: Plan exhausted
+[WARN] [1733882055.287061]: Plan exhausted
+[WARN] [1733882055.327050]: Plan exhausted
+[WARN] [1733882055.367044]: Plan exhausted
+[WARN] [1733882055.407220]: Plan exhausted
+[WARN] [1733882055.447047]: Plan exhausted
+[WARN] [1733882055.487638]: Plan exhausted
+[WARN] [1733882055.527050]: Plan exhausted
+[WARN] [1733882055.567115]: Plan exhausted
+[WARN] [1733882055.607057]: Plan exhausted
+[WARN] [1733882055.647342]: Plan exhausted
+[WARN] [1733882055.687266]: Plan exhausted
+[WARN] [1733882055.727048]: Plan exhausted
+[WARN] [1733882055.767036]: Plan exhausted
+[WARN] [1733882055.807041]: Plan exhausted
+[WARN] [1733882055.847427]: Plan exhausted
+[WARN] [1733882055.887833]: Plan exhausted
+[WARN] [1733882055.927052]: Plan exhausted
+[WARN] [1733882055.967049]: Plan exhausted
+[WARN] [1733882056.007052]: Plan exhausted
+[WARN] [1733882056.047083]: Plan exhausted
+[WARN] [1733882056.087589]: Plan exhausted
+[WARN] [1733882056.127044]: Plan exhausted
+[WARN] [1733882056.167048]: Plan exhausted
+[WARN] [1733882056.207262]: Plan exhausted
+[WARN] [1733882056.247047]: Plan exhausted
+[WARN] [1733882056.287124]: Plan exhausted
+[WARN] [1733882056.327050]: Plan exhausted
+[WARN] [1733882056.367157]: Plan exhausted
+[WARN] [1733882056.407021]: Plan exhausted
+[WARN] [1733882056.447050]: Plan exhausted
+[WARN] [1733882056.487767]: Plan exhausted
+[WARN] [1733882056.527117]: Plan exhausted
+[WARN] [1733882056.567130]: Plan exhausted
+[WARN] [1733882056.607051]: Plan exhausted
+[WARN] [1733882056.647138]: Plan exhausted
+[WARN] [1733882056.687124]: Plan exhausted
+[WARN] [1733882056.727053]: Plan exhausted
+[WARN] [1733882056.767150]: Plan exhausted
+[WARN] [1733882056.807051]: Plan exhausted
+[WARN] [1733882056.847173]: Plan exhausted
+[WARN] [1733882056.887062]: Plan exhausted
+[WARN] [1733882056.927035]: Plan exhausted
+[WARN] [1733882056.967050]: Plan exhausted
+[WARN] [1733882057.007113]: Plan exhausted
+[WARN] [1733882057.047042]: Plan exhausted
+[WARN] [1733882057.087055]: Plan exhausted
+[WARN] [1733882057.127055]: Plan exhausted
+[WARN] [1733882057.167049]: Plan exhausted
+[WARN] [1733882057.207051]: Plan exhausted
+[WARN] [1733882057.247331]: Plan exhausted
+[WARN] [1733882057.287073]: Plan exhausted
+[WARN] [1733882057.327050]: Plan exhausted
+[WARN] [1733882057.367083]: Plan exhausted
+[WARN] [1733882057.407049]: Plan exhausted
+[WARN] [1733882057.447057]: Plan exhausted
+[WARN] [1733882057.487026]: Plan exhausted
+[WARN] [1733882057.527054]: Plan exhausted
+[WARN] [1733882057.567045]: Plan exhausted
+[WARN] [1733882057.607049]: Plan exhausted
+[WARN] [1733882057.647047]: Plan exhausted
+[WARN] [1733882057.687011]: Plan exhausted
+[WARN] [1733882057.727017]: Plan exhausted
+[WARN] [1733882057.767211]: Plan exhausted
+[WARN] [1733882057.807042]: Plan exhausted
+[WARN] [1733882057.847021]: Plan exhausted
+[WARN] [1733882057.887207]: Plan exhausted
+[WARN] [1733882057.927228]: Plan exhausted
+[WARN] [1733882057.967051]: Plan exhausted
+[WARN] [1733882058.007057]: Plan exhausted
+[WARN] [1733882058.047322]: Plan exhausted
+[WARN] [1733882058.087226]: Plan exhausted
+[WARN] [1733882058.127048]: Plan exhausted
+[WARN] [1733882058.167130]: Plan exhausted
+[WARN] [1733882058.207052]: Plan exhausted
+[WARN] [1733882058.247214]: Plan exhausted
+[WARN] [1733882058.287363]: Plan exhausted
+[WARN] [1733882058.327049]: Plan exhausted
+[WARN] [1733882058.367048]: Plan exhausted
+[WARN] [1733882058.407050]: Plan exhausted
+[WARN] [1733882058.447052]: Plan exhausted
+[WARN] [1733882058.487058]: Plan exhausted
+[WARN] [1733882058.527298]: Plan exhausted
+[WARN] [1733882058.567122]: Plan exhausted
+[WARN] [1733882058.607055]: Plan exhausted
+[WARN] [1733882058.647049]: Plan exhausted
+[WARN] [1733882058.687101]: Plan exhausted
+[WARN] [1733882058.727047]: Plan exhausted
+[WARN] [1733882058.767135]: Plan exhausted
+[WARN] [1733882058.807058]: Plan exhausted
+[WARN] [1733882058.847335]: Plan exhausted
+[WARN] [1733882058.887022]: Plan exhausted
+[WARN] [1733882058.927045]: Plan exhausted
+[WARN] [1733882058.967049]: Plan exhausted
+[WARN] [1733882059.007245]: Plan exhausted
+[WARN] [1733882059.047005]: Plan exhausted
+[WARN] [1733882059.087120]: Plan exhausted
+[WARN] [1733882059.127012]: Plan exhausted
+[WARN] [1733882059.167065]: Plan exhausted
+[WARN] [1733882059.207019]: Plan exhausted
+[WARN] [1733882059.247036]: Plan exhausted
+[WARN] [1733882059.287063]: Plan exhausted
+[WARN] [1733882059.327055]: Plan exhausted
+[WARN] [1733882059.367099]: Plan exhausted
+[WARN] [1733882059.407047]: Plan exhausted
+[WARN] [1733882059.447119]: Plan exhausted
+[WARN] [1733882059.487020]: Plan exhausted
+[WARN] [1733882059.527021]: Plan exhausted
+[WARN] [1733882059.567276]: Plan exhausted
+[WARN] [1733882059.607046]: Plan exhausted
+[WARN] [1733882059.647049]: Plan exhausted
+[WARN] [1733882059.687052]: Plan exhausted
+[WARN] [1733882059.727055]: Plan exhausted
+[WARN] [1733882059.767051]: Plan exhausted
+[WARN] [1733882059.807057]: Plan exhausted
+[WARN] [1733882059.847101]: Plan exhausted
+[WARN] [1733882059.887052]: Plan exhausted
+[WARN] [1733882059.927545]: Plan exhausted
+[WARN] [1733882059.967116]: Plan exhausted
+[WARN] [1733882060.007119]: Plan exhausted
+[WARN] [1733882060.047050]: Plan exhausted
+[WARN] [1733882060.087050]: Plan exhausted
+[WARN] [1733882060.127049]: Plan exhausted
+[WARN] [1733882060.167005]: Plan exhausted
+[WARN] [1733882060.207313]: Plan exhausted
+[WARN] [1733882060.247017]: Plan exhausted
+[WARN] [1733882060.287113]: Plan exhausted
+[WARN] [1733882060.327050]: Plan exhausted
+[WARN] [1733882060.367122]: Plan exhausted
+[WARN] [1733882060.407054]: Plan exhausted
+[WARN] [1733882060.447045]: Plan exhausted
+[WARN] [1733882060.487054]: Plan exhausted
+[WARN] [1733882060.527052]: Plan exhausted
+[WARN] [1733882060.567052]: Plan exhausted
+[WARN] [1733882060.607050]: Plan exhausted
+[WARN] [1733882060.647051]: Plan exhausted
+[WARN] [1733882060.687018]: Plan exhausted
+[WARN] [1733882060.727043]: Plan exhausted
+[WARN] [1733882060.767119]: Plan exhausted
+[WARN] [1733882060.807206]: Plan exhausted
+[WARN] [1733882060.847048]: Plan exhausted
+[WARN] [1733882060.887264]: Plan exhausted
+[WARN] [1733882060.927234]: Plan exhausted
+[WARN] [1733882060.967058]: Plan exhausted
+[WARN] [1733882061.007091]: Plan exhausted
+[WARN] [1733882061.047144]: Plan exhausted
+[WARN] [1733882061.087239]: Plan exhausted
+[WARN] [1733882061.127246]: Plan exhausted
+[WARN] [1733882061.167158]: Plan exhausted
+[WARN] [1733882061.207054]: Plan exhausted
+[WARN] [1733882061.247165]: Plan exhausted
+[WARN] [1733882061.287024]: Plan exhausted
+[WARN] [1733882061.327046]: Plan exhausted
+[WARN] [1733882061.367183]: Plan exhausted
+[WARN] [1733882061.407129]: Plan exhausted
+[WARN] [1733882061.447046]: Plan exhausted
+[WARN] [1733882061.487052]: Plan exhausted
+[WARN] [1733882061.527047]: Plan exhausted
+[WARN] [1733882061.567054]: Plan exhausted
+[WARN] [1733882061.607161]: Plan exhausted
+[WARN] [1733882061.647009]: Plan exhausted
+[WARN] [1733882061.687433]: Plan exhausted
+[WARN] [1733882061.727046]: Plan exhausted
+[WARN] [1733882061.767049]: Plan exhausted
+[WARN] [1733882061.807220]: Plan exhausted
+[WARN] [1733882061.847046]: Plan exhausted
+[WARN] [1733882061.887051]: Plan exhausted
+[WARN] [1733882061.927050]: Plan exhausted
+[WARN] [1733882061.967053]: Plan exhausted
+[WARN] [1733882062.007053]: Plan exhausted
+[WARN] [1733882062.047162]: Plan exhausted
+[WARN] [1733882062.087059]: Plan exhausted
+[WARN] [1733882062.127038]: Plan exhausted
+[WARN] [1733882062.167129]: Plan exhausted
+[WARN] [1733882062.207045]: Plan exhausted
+[WARN] [1733882062.247299]: Plan exhausted
+[WARN] [1733882062.287050]: Plan exhausted
+[WARN] [1733882062.327052]: Plan exhausted
+[WARN] [1733882062.367054]: Plan exhausted
+[WARN] [1733882062.407196]: Plan exhausted
+[WARN] [1733882062.447055]: Plan exhausted
+[WARN] [1733882062.487156]: Plan exhausted
+[WARN] [1733882062.527073]: Plan exhausted
+[WARN] [1733882062.567137]: Plan exhausted
+[WARN] [1733882062.607140]: Plan exhausted
+[WARN] [1733882062.647053]: Plan exhausted
+[WARN] [1733882062.687056]: Plan exhausted
+[WARN] [1733882062.727053]: Plan exhausted
+[WARN] [1733882062.767071]: Plan exhausted
+[WARN] [1733882062.807082]: Plan exhausted
+[WARN] [1733882062.847049]: Plan exhausted
+[WARN] [1733882062.887068]: Plan exhausted
+[WARN] [1733882062.927056]: Plan exhausted
+[WARN] [1733882062.967074]: Plan exhausted
+[WARN] [1733882063.007052]: Plan exhausted
+[WARN] [1733882063.047042]: Plan exhausted
+[WARN] [1733882063.087048]: Plan exhausted
+[WARN] [1733882063.127049]: Plan exhausted
+[WARN] [1733882063.167047]: Plan exhausted
+[WARN] [1733882063.207056]: Plan exhausted
+[WARN] [1733882063.247158]: Plan exhausted
+[WARN] [1733882063.287093]: Plan exhausted
+[WARN] [1733882063.327050]: Plan exhausted
+[WARN] [1733882063.367152]: Plan exhausted
+[WARN] [1733882063.407056]: Plan exhausted
+[WARN] [1733882063.447058]: Plan exhausted
+[WARN] [1733882063.487025]: Plan exhausted
+[WARN] [1733882063.527024]: Plan exhausted
+[WARN] [1733882063.567053]: Plan exhausted
+[WARN] [1733882063.607054]: Plan exhausted
+[WARN] [1733882063.647300]: Plan exhausted
+[WARN] [1733882063.687048]: Plan exhausted
+[WARN] [1733882063.727015]: Plan exhausted
+[WARN] [1733882063.767130]: Plan exhausted
+[WARN] [1733882063.807046]: Plan exhausted
+[WARN] [1733882063.847275]: Plan exhausted
+[WARN] [1733882063.887052]: Plan exhausted
+[WARN] [1733882063.927107]: Plan exhausted
+[WARN] [1733882063.967128]: Plan exhausted
+[WARN] [1733882064.007052]: Plan exhausted
+[WARN] [1733882064.047224]: Plan exhausted
+[WARN] [1733882064.087026]: Plan exhausted
+[WARN] [1733882064.127050]: Plan exhausted
+[WARN] [1733882064.167024]: Plan exhausted
+[WARN] [1733882064.207025]: Plan exhausted
+[WARN] [1733882064.247044]: Plan exhausted
+[WARN] [1733882064.287042]: Plan exhausted
+[WARN] [1733882064.327054]: Plan exhausted
+[WARN] [1733882064.367053]: Plan exhausted
+[WARN] [1733882064.407223]: Plan exhausted
+[WARN] [1733882064.447106]: Plan exhausted
+[WARN] [1733882064.487052]: Plan exhausted
+[WARN] [1733882064.527056]: Plan exhausted
+[WARN] [1733882064.567079]: Plan exhausted
+[WARN] [1733882064.607060]: Plan exhausted
+[WARN] [1733882064.647050]: Plan exhausted
+[WARN] [1733882064.687174]: Plan exhausted
+[WARN] [1733882064.727053]: Plan exhausted
+[WARN] [1733882064.767020]: Plan exhausted
+[WARN] [1733882064.807027]: Plan exhausted
+[WARN] [1733882064.847050]: Plan exhausted
+[WARN] [1733882064.887045]: Plan exhausted
+[WARN] [1733882064.927509]: Plan exhausted
+[WARN] [1733882064.967138]: Plan exhausted
+[WARN] [1733882065.007050]: Plan exhausted
+[WARN] [1733882065.047022]: Plan exhausted
+[WARN] [1733882065.087052]: Plan exhausted
+[WARN] [1733882065.127049]: Plan exhausted
+[WARN] [1733882065.167045]: Plan exhausted
+[WARN] [1733882065.207147]: Plan exhausted
+[WARN] [1733882065.247191]: Plan exhausted
+[WARN] [1733882065.287047]: Plan exhausted
+[WARN] [1733882065.327043]: Plan exhausted
+[WARN] [1733882065.367355]: Plan exhausted
+[WARN] [1733882065.407120]: Plan exhausted
+[WARN] [1733882065.447181]: Plan exhausted
+[WARN] [1733882065.487173]: Plan exhausted
+[WARN] [1733882065.527201]: Plan exhausted
+[WARN] [1733882065.567048]: Plan exhausted
+[WARN] [1733882065.607166]: Plan exhausted
+[WARN] [1733882065.647006]: Plan exhausted
+[WARN] [1733882065.687023]: Plan exhausted
+[WARN] [1733882065.727056]: Plan exhausted
+[WARN] [1733882065.767043]: Plan exhausted
+[WARN] [1733882065.807048]: Plan exhausted
+[WARN] [1733882065.847017]: Plan exhausted
+[WARN] [1733882065.887288]: Plan exhausted
+[WARN] [1733882065.927184]: Plan exhausted
+[WARN] [1733882065.967075]: Plan exhausted
+[WARN] [1733882066.007171]: Plan exhausted
+[WARN] [1733882066.047332]: Plan exhausted
+[WARN] [1733882066.087051]: Plan exhausted
+[WARN] [1733882066.127206]: Plan exhausted
+[WARN] [1733882066.167326]: Plan exhausted
+[WARN] [1733882066.207052]: Plan exhausted
+[WARN] [1733882066.247040]: Plan exhausted
+[WARN] [1733882066.287051]: Plan exhausted
+[WARN] [1733882066.327026]: Plan exhausted
+[WARN] [1733882066.367097]: Plan exhausted
+[WARN] [1733882066.407017]: Plan exhausted
+[WARN] [1733882066.447052]: Plan exhausted
+[WARN] [1733882066.487055]: Plan exhausted
+[WARN] [1733882066.527056]: Plan exhausted
+[WARN] [1733882066.567098]: Plan exhausted
+[WARN] [1733882066.607184]: Plan exhausted
+[WARN] [1733882066.647047]: Plan exhausted
+[WARN] [1733882066.687049]: Plan exhausted
+[WARN] [1733882066.727019]: Plan exhausted
+[WARN] [1733882066.767239]: Plan exhausted
+[WARN] [1733882066.807016]: Plan exhausted
+[WARN] [1733882066.847045]: Plan exhausted
+[WARN] [1733882066.887052]: Plan exhausted
+[WARN] [1733882066.927155]: Plan exhausted
+[WARN] [1733882066.967067]: Plan exhausted
+[WARN] [1733882067.007022]: Plan exhausted
+[WARN] [1733882067.047061]: Plan exhausted
+[WARN] [1733882067.087224]: Plan exhausted
+[WARN] [1733882067.127199]: Plan exhausted
+[WARN] [1733882067.167049]: Plan exhausted
+[WARN] [1733882067.207051]: Plan exhausted
+[WARN] [1733882067.247069]: Plan exhausted
+[WARN] [1733882067.287317]: Plan exhausted
+[WARN] [1733882067.327368]: Plan exhausted
+[WARN] [1733882067.367091]: Plan exhausted
+[WARN] [1733882067.407023]: Plan exhausted
+[WARN] [1733882067.447050]: Plan exhausted
+[WARN] [1733882067.487056]: Plan exhausted
+[WARN] [1733882067.527055]: Plan exhausted
+[WARN] [1733882067.578561]: Plan exhausted
+[WARN] [1733882067.607018]: Plan exhausted
+[WARN] [1733882067.647185]: Plan exhausted
+Exception in thread Thread-62:
+Traceback (most recent call last):
+  File "/home/leonmkim/.pyenv/versions/3.10.14/lib/python3.10/threading.py", line 1016, in _bootstrap_inner
+    self.run()
+  File "/opt/ros/noetic/lib/python3/dist-packages/rospy/timer.py", line 237, in run
+    self._callback(TimerEvent(last_expected, last_real, current_expected, current_real, last_duration))
+  File "/home/leonmkim/fish_leon/FISH/video.py", line 78, in ros_record_cb
+    self.process.stdin.write(frame.tobytes())
+BrokenPipeError: [Errno 32] Broken pipe
+[WARN] [1733882067.687189]: Plan exhausted
+[WARN] [1733882067.727179]: Plan exhausted
+[WARN] [1733882067.767038]: Plan exhausted
+[WARN] [1733882067.807042]: Plan exhausted
+[WARN] [1733882067.847068]: Plan exhausted
+[WARN] [1733882067.887222]: Plan exhausted
+[WARN] [1733882067.927459]: Plan exhausted
+[WARN] [1733882067.967060]: Plan exhausted
+[WARN] [1733882068.007040]: Plan exhausted
+[WARN] [1733882068.047081]: Plan exhausted
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit
+[pub_text-1] killing on exit
+program exiting gracefully
+proceeding to start episode!
+[INFO] [1733882069.242753]: Saving video...
+[INFO] [1733882069.243578]: Video saved!

205324/wandb/run-20241210_205330-52nddla7/files/requirements.txt

@@ -0,0 +1,339 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.2
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0

205324/wandb/run-20241210_205330-52nddla7/files/wandb-metadata.json

@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2024-12-11T01:53:31.224621",
+    "startedAt": "2024-12-11T01:53:30.842354",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "4f7c0421eb84a7caf6bf5670f68777523362676f"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 4312.5025,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 4500.019,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4500.002,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4500.007,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.999,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.943,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 365.2335090637207
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71596908569336
+    }
+}

205324/wandb/run-20241210_205330-52nddla7/files/wandb-summary.json

@@ -0,0 +1 @@
+{"eval/video": {"_type": "video-file", "sha256": "ff4edecb6a17ea039fbf12ba8ab1396b6c7798607c09ee107432df34eedac9aa", "size": 622320, "path": "media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4"}, "global_step": 1, "_timestamp": 1733882069.2541587, "_runtime": 58.40333962440491, "_step": 0, "_wandb": {"runtime": 58}}

205324/wandb/run-20241210_205330-52nddla7/logs/debug-internal.log

@@ -0,0 +1,327 @@
+2024-12-10 20:53:30,850 INFO    StreamThr :598006 [internal.py:wandb_internal():85] W&B internal server running at pid: 598006, started at: 2024-12-10 20:53:30.850039
+2024-12-10 20:53:30,851 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status
+2024-12-10 20:53:30,863 INFO    WriterThread:598006 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/run-52nddla7.wandb
+2024-12-10 20:53:30,863 DEBUG   SenderThread:598006 [sender.py:send():379] send: header
+2024-12-10 20:53:30,875 DEBUG   SenderThread:598006 [sender.py:send():379] send: run
+2024-12-10 20:53:31,147 INFO    SenderThread:598006 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files
+2024-12-10 20:53:31,147 INFO    SenderThread:598006 [sender.py:_start_run_threads():1188] run started: 52nddla7 with start time 1733882010.850819
+2024-12-10 20:53:31,161 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: check_version
+2024-12-10 20:53:31,161 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: check_version
+2024-12-10 20:53:31,206 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: run_start
+2024-12-10 20:53:31,211 DEBUG   HandlerThread:598006 [system_info.py:__init__():26] System info init
+2024-12-10 20:53:31,211 DEBUG   HandlerThread:598006 [system_info.py:__init__():41] System info init done
+2024-12-10 20:53:31,211 INFO    HandlerThread:598006 [system_monitor.py:start():194] Starting system monitor
+2024-12-10 20:53:31,211 INFO    SystemMonitor:598006 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-10 20:53:31,212 INFO    HandlerThread:598006 [system_monitor.py:probe():214] Collecting system info
+2024-12-10 20:53:31,212 INFO    SystemMonitor:598006 [interfaces.py:start():188] Started cpu monitoring
+2024-12-10 20:53:31,212 INFO    SystemMonitor:598006 [interfaces.py:start():188] Started disk monitoring
+2024-12-10 20:53:31,213 INFO    SystemMonitor:598006 [interfaces.py:start():188] Started gpu monitoring
+2024-12-10 20:53:31,213 INFO    SystemMonitor:598006 [interfaces.py:start():188] Started memory monitoring
+2024-12-10 20:53:31,214 INFO    SystemMonitor:598006 [interfaces.py:start():188] Started network monitoring
+2024-12-10 20:53:31,224 DEBUG   HandlerThread:598006 [system_info.py:probe():152] Probing system
+2024-12-10 20:53:31,226 DEBUG   HandlerThread:598006 [system_info.py:_probe_git():137] Probing git
+2024-12-10 20:53:31,231 DEBUG   HandlerThread:598006 [system_info.py:_probe_git():145] Probing git done
+2024-12-10 20:53:31,231 DEBUG   HandlerThread:598006 [system_info.py:probe():200] Probing system done
+2024-12-10 20:53:31,231 DEBUG   HandlerThread:598006 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-11T01:53:31.224621', 'startedAt': '2024-12-11T01:53:30.842354', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '4f7c0421eb84a7caf6bf5670f68777523362676f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4312.5025, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4500.019, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.007, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.943, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 365.2335090637207}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71596908569336}}
+2024-12-10 20:53:31,231 INFO    HandlerThread:598006 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-10 20:53:31,231 INFO    HandlerThread:598006 [system_monitor.py:probe():227] Publishing system info
+2024-12-10 20:53:31,231 DEBUG   HandlerThread:598006 [system_info.py:_save_code():44] Saving code
+2024-12-10 20:53:31,233 DEBUG   HandlerThread:598006 [system_info.py:_save_code():67] Saving code done
+2024-12-10 20:53:31,233 DEBUG   HandlerThread:598006 [system_info.py:_save_patches():84] Saving git patches
+2024-12-10 20:53:31,305 DEBUG   HandlerThread:598006 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-10 20:53:31,306 INFO    HandlerThread:598006 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-10 20:53:31,309 DEBUG   SenderThread:598006 [sender.py:send():379] send: files
+2024-12-10 20:53:31,310 INFO    SenderThread:598006 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-10 20:53:31,310 INFO    SenderThread:598006 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-10 20:53:31,310 INFO    SenderThread:598006 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-10 20:53:31,570 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-10 20:53:31,570 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: python_packages
+2024-12-10 20:53:31,570 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-10 20:53:31,571 INFO    wandb-upload_0:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/oqlixunj-wandb-metadata.json
+2024-12-10 20:53:31,571 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:31,572 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: stop_status
+2024-12-10 20:53:31,573 INFO    HandlerThread:598006 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/tb"
+  save: true
+}
+control {
+  relay_id: "5d813b8bb5a547c8896a7e03615dca84"
+}
+_info {
+  stream_id: "52nddla7"
+}
+
+2024-12-10 20:53:31,574 DEBUG   HandlerThread:598006 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-10 20:53:31,605 INFO    wandb-upload_1:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/6yv6eek3-code/FISH/eval_robot.py
+2024-12-10 20:53:31,648 INFO    wandb-upload_2:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/w33l6u66-diff.patch
+2024-12-10 20:53:31,702 DEBUG   SenderThread:598006 [sender.py:send():379] send: telemetry
+2024-12-10 20:53:31,702 DEBUG   SenderThread:598006 [sender.py:send():379] send: tbrecord
+2024-12-10 20:53:31,703 DEBUG   SenderThread:598006 [sender.py:send():379] send: files
+2024-12-10 20:53:31,703 INFO    SenderThread:598006 [sender.py:_save_file():1454] saving file events.out.tfevents.1733882011.leonmkim-ROG-Strix-G15CS-G15CS.597884.0 with policy live
+2024-12-10 20:53:31,907 INFO    wandb-upload_0:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/uat8jiac-events.out.tfevents.1733882011.leonmkim-ROG-Strix-G15CS-G15CS.597884.0
+2024-12-10 20:53:32,149 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/events.out.tfevents.1733882011.leonmkim-ROG-Strix-G15CS-G15CS.597884.0
+2024-12-10 20:53:32,149 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/code/FISH/eval_robot.py
+2024-12-10 20:53:32,149 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-metadata.json
+2024-12-10 20:53:32,149 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:53:32,149 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/requirements.txt
+2024-12-10 20:53:32,149 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/diff.patch
+2024-12-10 20:53:32,149 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/code/FISH
+2024-12-10 20:53:32,149 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/code
+2024-12-10 20:53:32,571 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:33,571 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:34,148 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:53:34,571 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:35,547 DEBUG   SenderThread:598006 [sender.py:send():379] send: config
+2024-12-10 20:53:35,590 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:36,571 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:53:36,572 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:37,572 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:38,149 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:53:38,572 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:39,572 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:40,150 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:53:40,572 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:41,572 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:42,082 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:53:42,150 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:53:42,572 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:43,572 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:44,151 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:53:44,573 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:45,573 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:46,151 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:53:46,570 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-10 20:53:46,570 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: stop_status
+2024-12-10 20:53:46,614 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:47,573 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:47,640 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:53:48,573 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:49,573 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:50,573 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:51,574 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:52,574 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:53,571 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:53:53,574 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:54,574 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:55,574 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:56,574 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:57,574 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:58,571 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:53:58,574 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:53:59,575 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:00,575 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:01,570 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-10 20:54:01,570 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: stop_status
+2024-12-10 20:54:01,613 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:02,575 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:03,575 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:03,722 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:54:04,156 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/config.yaml
+2024-12-10 20:54:04,575 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:05,575 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:06,575 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:07,576 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:08,576 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:09,550 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:54:09,576 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:10,158 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:10,576 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:11,576 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:12,159 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:12,577 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:13,577 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:14,159 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:14,567 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:54:14,577 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:15,160 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:15,577 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:16,570 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-10 20:54:16,570 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: stop_status
+2024-12-10 20:54:16,577 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:17,160 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:17,577 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:18,579 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:19,161 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:19,567 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:54:19,577 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:20,577 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:21,161 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:21,577 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:22,578 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:23,162 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:23,578 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:24,578 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:24,607 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:54:25,162 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:25,578 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:26,578 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:27,163 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:27,579 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:28,579 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:29,164 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:29,254 DEBUG   SenderThread:598006 [sender.py:send():379] send: files
+2024-12-10 20:54:29,254 INFO    SenderThread:598006 [sender.py:_save_file():1454] saving file media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4 with policy now
+2024-12-10 20:54:29,256 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-10 20:54:29,257 DEBUG   SenderThread:598006 [sender.py:send():379] send: history
+2024-12-10 20:54:29,257 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: summary_record
+2024-12-10 20:54:29,257 INFO    SenderThread:598006 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-10 20:54:29,258 DEBUG   SenderThread:598006 [sender.py:send():379] send: exit
+2024-12-10 20:54:29,258 INFO    SenderThread:598006 [sender.py:send_exit():586] handling exit code: 0
+2024-12-10 20:54:29,259 INFO    SenderThread:598006 [sender.py:send_exit():588] handling runtime: 58
+2024-12-10 20:54:29,259 INFO    SenderThread:598006 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-10 20:54:29,259 INFO    SenderThread:598006 [sender.py:send_exit():594] send defer
+2024-12-10 20:54:29,259 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:29,259 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-10 20:54:29,259 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:29,259 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-10 20:54:29,259 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 1
+2024-12-10 20:54:29,259 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:29,259 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-10 20:54:29,260 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:29,260 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-10 20:54:29,260 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 2
+2024-12-10 20:54:29,260 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:29,260 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-10 20:54:29,260 INFO    HandlerThread:598006 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-10 20:54:29,260 DEBUG   SystemMonitor:598006 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-10 20:54:29,260 DEBUG   SystemMonitor:598006 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-10 20:54:29,260 DEBUG   SystemMonitor:598006 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-10 20:54:29,269 INFO    HandlerThread:598006 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-10 20:54:29,269 INFO    HandlerThread:598006 [interfaces.py:finish():200] Joined disk monitor
+2024-12-10 20:54:29,319 INFO    HandlerThread:598006 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-10 20:54:29,319 INFO    HandlerThread:598006 [interfaces.py:finish():200] Joined memory monitor
+2024-12-10 20:54:29,320 INFO    HandlerThread:598006 [interfaces.py:finish():200] Joined network monitor
+2024-12-10 20:54:29,320 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:29,320 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-10 20:54:29,320 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 3
+2024-12-10 20:54:29,320 DEBUG   SenderThread:598006 [sender.py:send():379] send: stats
+2024-12-10 20:54:29,320 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:29,320 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-10 20:54:29,321 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:29,321 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-10 20:54:29,321 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 4
+2024-12-10 20:54:29,321 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:29,321 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-10 20:54:29,518 INFO    wandb-upload_2:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/0kw42edo-media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4
+2024-12-10 20:54:30,164 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:30,164 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4
+2024-12-10 20:54:30,164 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-summary.json
+2024-12-10 20:54:30,165 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/media/videos
+2024-12-10 20:54:30,165 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/media/videos/eval
+2024-12-10 20:54:30,165 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/media
+2024-12-10 20:54:30,523 INFO    wandb-upload_0:598006 [upload_job.py:push():130] Uploaded file /tmp/tmptbytljn2wandb/ks8s91l3-media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4
+2024-12-10 20:54:35,581 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-10 20:54:35,581 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:54:35,581 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-10 20:54:35,581 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:54:35,581 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:35,581 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-10 20:54:35,581 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 5
+2024-12-10 20:54:35,581 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit
+2024-12-10 20:54:35,582 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:35,582 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-10 20:54:35,582 DEBUG   SenderThread:598006 [sender.py:send():379] send: summary
+2024-12-10 20:54:35,582 INFO    SenderThread:598006 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-10 20:54:35,582 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:35,582 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-10 20:54:35,582 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 6
+2024-12-10 20:54:35,583 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:35,583 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-10 20:54:35,583 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:35,583 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-10 20:54:35,583 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 7
+2024-12-10 20:54:35,583 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: status_report
+2024-12-10 20:54:35,583 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:35,583 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-10 20:54:35,583 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:35,583 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-10 20:54:36,166 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-summary.json
+2024-12-10 20:54:36,260 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-10 20:54:37,110 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 8
+2024-12-10 20:54:37,110 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit
+2024-12-10 20:54:37,110 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:37,110 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-10 20:54:37,111 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:37,111 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-10 20:54:37,111 INFO    SenderThread:598006 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-10 20:54:37,113 INFO    SenderThread:598006 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-10 20:54:37,127 INFO    SenderThread:598006 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-10 20:54:37,136 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 9
+2024-12-10 20:54:37,136 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:37,136 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-10 20:54:37,137 DEBUG   SenderThread:598006 [sender.py:send():379] send: artifact
+2024-12-10 20:54:37,166 INFO    Thread-12 :598006 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:37,260 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-10 20:54:37,780 INFO    wandb-upload_1:598006 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_p_p260o
+2024-12-10 20:54:37,783 INFO    wandb-upload_0:598006 [upload_job.py:push():85] Skipped uploading /tmp/tmppldao70p/wandb-job.json
+2024-12-10 20:54:37,954 INFO    wandb-upload_2:598006 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4q7vuxho
+2024-12-10 20:54:38,567 INFO    SenderThread:598006 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM2Njc2MTczOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM2Njc1OTg1Ng==', 'versionIndex': 336}}}
+2024-12-10 20:54:38,567 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:38,567 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-10 20:54:38,567 INFO    SenderThread:598006 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-10 20:54:39,167 INFO    SenderThread:598006 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files
+2024-12-10 20:54:39,168 INFO    SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/config.yaml config.yaml
+2024-12-10 20:54:39,168 INFO    SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-summary.json wandb-summary.json
+2024-12-10 20:54:39,168 INFO    SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/events.out.tfevents.1733882011.leonmkim-ROG-Strix-G15CS-G15CS.597884.0 events.out.tfevents.1733882011.leonmkim-ROG-Strix-G15CS-G15CS.597884.0
+2024-12-10 20:54:39,168 INFO    SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/requirements.txt requirements.txt
+2024-12-10 20:54:39,168 INFO    SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-metadata.json wandb-metadata.json
+2024-12-10 20:54:39,168 INFO    SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log output.log
+2024-12-10 20:54:39,173 INFO    SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/diff.patch diff.patch
+2024-12-10 20:54:39,173 INFO    SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-10 20:54:39,174 INFO    SenderThread:598006 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4 media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4
+2024-12-10 20:54:39,175 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 10
+2024-12-10 20:54:39,175 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit
+2024-12-10 20:54:39,175 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:39,175 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-10 20:54:39,175 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:39,175 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-10 20:54:39,175 INFO    SenderThread:598006 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-10 20:54:39,261 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-10 20:54:39,261 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit
+2024-12-10 20:54:39,363 INFO    wandb-upload_0:598006 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/wandb-summary.json
+2024-12-10 20:54:39,397 INFO    wandb-upload_2:598006 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/requirements.txt
+2024-12-10 20:54:39,416 INFO    wandb-upload_1:598006 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/config.yaml
+2024-12-10 20:54:39,447 INFO    wandb-upload_3:598006 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/files/output.log
+2024-12-10 20:54:39,647 INFO    Thread-11 (_thread_body):598006 [sender.py:transition_state():614] send defer: 11
+2024-12-10 20:54:39,647 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:39,647 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-10 20:54:39,648 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:39,648 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-10 20:54:39,648 INFO    SenderThread:598006 [file_pusher.py:join():175] waiting for file pusher
+2024-12-10 20:54:39,648 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 12
+2024-12-10 20:54:39,648 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:39,648 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-10 20:54:39,648 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:39,649 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-10 20:54:39,649 INFO    SenderThread:598006 [file_stream.py:finish():601] file stream finish called
+2024-12-10 20:54:39,709 INFO    SenderThread:598006 [file_stream.py:finish():605] file stream finish is done
+2024-12-10 20:54:39,709 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 13
+2024-12-10 20:54:39,709 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:39,709 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-10 20:54:39,709 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:39,709 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-10 20:54:39,709 INFO    SenderThread:598006 [sender.py:transition_state():614] send defer: 14
+2024-12-10 20:54:39,709 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: defer
+2024-12-10 20:54:39,709 INFO    HandlerThread:598006 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-10 20:54:39,710 DEBUG   SenderThread:598006 [sender.py:send():379] send: final
+2024-12-10 20:54:39,710 DEBUG   SenderThread:598006 [sender.py:send():379] send: footer
+2024-12-10 20:54:39,710 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: defer
+2024-12-10 20:54:39,710 INFO    SenderThread:598006 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-10 20:54:39,710 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-10 20:54:39,710 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: server_info
+2024-12-10 20:54:39,710 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-10 20:54:39,711 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit
+2024-12-10 20:54:39,711 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: server_info
+2024-12-10 20:54:39,712 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-10 20:54:39,713 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-10 20:54:39,713 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-10 20:54:39,793 DEBUG   SenderThread:598006 [sender.py:send_request():406] send_request: poll_exit
+2024-12-10 20:54:39,794 INFO    MainThread:598006 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-10 20:54:39,794 INFO    MainThread:598006 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-10 20:54:39,794 INFO    MainThread:598006 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-10 20:54:39,794 DEBUG   HandlerThread:598006 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-10 20:54:39,794 INFO    HandlerThread:598006 [handler.py:finish():882] shutting down handler
+2024-12-10 20:54:40,712 INFO    WriterThread:598006 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/run-52nddla7.wandb
+2024-12-10 20:54:40,794 INFO    SenderThread:598006 [sender.py:finish():1615] shutting down sender
+2024-12-10 20:54:40,794 INFO    SenderThread:598006 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-10 20:54:40,794 INFO    SenderThread:598006 [file_pusher.py:join():175] waiting for file pusher

205324/wandb/run-20241210_205330-52nddla7/logs/debug.log

@@ -0,0 +1,31 @@
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_setup.py:_flush():76] Configure stats pid to 597884
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/settings
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/logs/debug.log
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/wandb/run-20241210_205330-52nddla7/logs/debug-internal.log
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_init.py:init():569] calling init triggers
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '265626', 'wandb_notes': '265626_req_265827', 'eval': True, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim_left_to_right'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'obs_shape': [5, 240, 320], 'action_shape': [36, 7], 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'true_action_history': False, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt'}
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_init.py:init():619] starting backend
+2024-12-10 20:53:30,845 INFO    MainThread:597884 [wandb_init.py:init():623] setting up manager
+2024-12-10 20:53:30,849 INFO    MainThread:597884 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-10 20:53:30,850 INFO    MainThread:597884 [wandb_init.py:init():631] backend started and connected
+2024-12-10 20:53:30,862 INFO    MainThread:597884 [wandb_init.py:init():720] updated telemetry
+2024-12-10 20:53:30,871 INFO    MainThread:597884 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-10 20:53:31,160 INFO    MainThread:597884 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-10 20:53:31,200 INFO    MainThread:597884 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.0 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-10 20:53:31,201 INFO    MainThread:597884 [wandb_init.py:init():804] starting run threads in backend
+2024-12-10 20:53:31,571 INFO    MainThread:597884 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-10 20:53:31,571 INFO    MainThread:597884 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-10 20:53:31,571 INFO    MainThread:597884 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-10 20:53:31,571 INFO    MainThread:597884 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-10 20:53:31,573 INFO    MainThread:597884 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-10 20:53:31,573 INFO    MainThread:597884 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/205324/tb, True
+2024-12-10 20:53:35,546 INFO    MainThread:597884 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'}
+2024-12-10 20:54:40,795 WARNING MsgRouterThr:597884 [router.py:message_loop():77] message_loop has been closed