Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,18 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/1_eval_1_528c8c53973fc263e4c2.mp4 filter=lfs diff=lfs merge=lfs -text
+211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/4_eval_4_85f8c5555fdc1daec6f7.mp4 filter=lfs diff=lfs merge=lfs -text
+211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/2_eval_2_2f040d8fedc4c652052d.mp4 filter=lfs diff=lfs merge=lfs -text
+211647/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+211647/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+211647/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+211647/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/3_eval_3_d96484f372adc32d22ea.mp4 filter=lfs diff=lfs merge=lfs -text
+211647/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/0_eval_0_607b63808c40640027f8.mp4 filter=lfs diff=lfs merge=lfs -text
+211647/episode_rosbags/episode_3_2024-12-17-21-19-23.bag filter=lfs diff=lfs merge=lfs -text
+211647/episode_rosbags/episode_4_2024-12-17-21-19-58.bag filter=lfs diff=lfs merge=lfs -text
+211647/episode_rosbags/episode_2_2024-12-17-21-18-48.bag filter=lfs diff=lfs merge=lfs -text
+211647/episode_rosbags/episode_1_2024-12-17-21-17-53.bag filter=lfs diff=lfs merge=lfs -text
+211647/episode_rosbags/episode_0_2024-12-17-21-17-13.bag filter=lfs diff=lfs merge=lfs -text

211645/.hydra/config.yaml

@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '508_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${now:%H%M%S}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

211645/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_policy
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211645
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

211645/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion

211645/eval_policy.log

@@ -0,0 +1,15 @@
+[2024-12-17 21:16:45,761][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-17 21:16:45,765][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2024-12-17 21:16:48,373][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+

211647/.hydra/config.yaml

@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '508_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${now:%H%M%S}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

211647/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

211647/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion

211647/episode_rosbags/aligned_depth_to_color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

211647/episode_rosbags/cam_tf_world.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256

211647/episode_rosbags/color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

211647/episode_rosbags/depth_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200

211647/episode_rosbags/episode_0_2024-12-17-21-17-13.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95fcdf567cd4c03d0be23e7d6361006385a31147338e5b44e222a673da560405
+size 1329666251

211647/episode_rosbags/episode_1_2024-12-17-21-17-53.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e0480e5d65177a82fce5cbd36d5794c627e2830055f1de236d5a49416fe248a
+size 802070163

211647/episode_rosbags/episode_2_2024-12-17-21-18-48.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c4f1d2c7f6dbe5227b4005625ec80370ba4f711a2a98a076630bf078768e38f
+size 592043733

211647/episode_rosbags/episode_3_2024-12-17-21-19-23.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09678d9606a0955d61b7d4facf27b8dc7c43e6d8cd4af590ec4823127eae308a
+size 573990786

211647/episode_rosbags/episode_4_2024-12-17-21-19-58.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e15fd75dad26bf5a5d9cb49f4fa3d690e42543bc335969252f79e32eee2d953a
+size 611352152

211647/eval_robot.log

@@ -0,0 +1,12 @@
+[2024-12-17 21:16:47,091][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-17 21:16:47,094][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+

211647/eval_video/0_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:607b63808c40640027f8da1ae9902a7aa7037aa7b73881ae52fcd560abc6131c
+size 1162092

211647/eval_video/1_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:528c8c53973fc263e4c239f72cdd4f717b7ebe1bafb47099a337bf475618128c
+size 734029

211647/eval_video/2_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f040d8fedc4c652052df87e67a190bae28ff42773fdd3024edaf607ac6a0bd0
+size 585337

211647/eval_video/3_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d96484f372adc32d22ea328440dc6fca791adad01df8190f1167d57835cf3588
+size 640584

211647/eval_video/4_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85f8c5555fdc1daec6f75870c681078310d2198c2dbcf62e2cdc8271016d199f
+size 670569

211647/tb/events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5df75a6a40c6241fb479e11bbfaf27ab7ab64092b846041761082a55ea86a419
+size 1103

211647/wandb/debug-internal.log

@@ -0,0 +1,650 @@
+2024-12-17 21:16:53,171 INFO    StreamThr :75002 [internal.py:wandb_internal():85] W&B internal server running at pid: 75002, started at: 2024-12-17 21:16:53.170951
+2024-12-17 21:16:53,172 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status
+2024-12-17 21:16:53,183 INFO    WriterThread:75002 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/run-14r6tvph.wandb
+2024-12-17 21:16:53,183 DEBUG   SenderThread:75002 [sender.py:send():379] send: header
+2024-12-17 21:16:53,191 DEBUG   SenderThread:75002 [sender.py:send():379] send: run
+2024-12-17 21:16:53,401 INFO    SenderThread:75002 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files
+2024-12-17 21:16:53,401 INFO    SenderThread:75002 [sender.py:_start_run_threads():1188] run started: 14r6tvph with start time 1734488213.172477
+2024-12-17 21:16:53,414 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: check_version
+2024-12-17 21:16:53,414 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: check_version
+2024-12-17 21:16:53,486 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: run_start
+2024-12-17 21:16:53,491 DEBUG   HandlerThread:75002 [system_info.py:__init__():26] System info init
+2024-12-17 21:16:53,491 DEBUG   HandlerThread:75002 [system_info.py:__init__():41] System info init done
+2024-12-17 21:16:53,491 INFO    HandlerThread:75002 [system_monitor.py:start():194] Starting system monitor
+2024-12-17 21:16:53,491 INFO    SystemMonitor:75002 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-17 21:16:53,491 INFO    HandlerThread:75002 [system_monitor.py:probe():214] Collecting system info
+2024-12-17 21:16:53,492 INFO    SystemMonitor:75002 [interfaces.py:start():188] Started cpu monitoring
+2024-12-17 21:16:53,492 INFO    SystemMonitor:75002 [interfaces.py:start():188] Started disk monitoring
+2024-12-17 21:16:53,493 INFO    SystemMonitor:75002 [interfaces.py:start():188] Started gpu monitoring
+2024-12-17 21:16:53,493 INFO    SystemMonitor:75002 [interfaces.py:start():188] Started memory monitoring
+2024-12-17 21:16:53,494 INFO    SystemMonitor:75002 [interfaces.py:start():188] Started network monitoring
+2024-12-17 21:16:53,503 DEBUG   HandlerThread:75002 [system_info.py:probe():152] Probing system
+2024-12-17 21:16:53,505 DEBUG   HandlerThread:75002 [system_info.py:_probe_git():137] Probing git
+2024-12-17 21:16:53,510 DEBUG   HandlerThread:75002 [system_info.py:_probe_git():145] Probing git done
+2024-12-17 21:16:53,510 DEBUG   HandlerThread:75002 [system_info.py:probe():200] Probing system done
+2024-12-17 21:16:53,511 DEBUG   HandlerThread:75002 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-18T02:16:53.503865', 'startedAt': '2024-12-18T02:16:53.161156', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '5f28b54373d7d41e4ebace380ff1080a8ffc4051'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3747.0996250000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4495.328, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.364, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 386.0898132324219}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2024-12-17 21:16:53,511 INFO    HandlerThread:75002 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-17 21:16:53,511 INFO    HandlerThread:75002 [system_monitor.py:probe():227] Publishing system info
+2024-12-17 21:16:53,511 DEBUG   HandlerThread:75002 [system_info.py:_save_code():44] Saving code
+2024-12-17 21:16:53,512 DEBUG   HandlerThread:75002 [system_info.py:_save_code():67] Saving code done
+2024-12-17 21:16:53,512 DEBUG   HandlerThread:75002 [system_info.py:_save_patches():84] Saving git patches
+2024-12-17 21:16:53,585 DEBUG   HandlerThread:75002 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-17 21:16:53,586 INFO    HandlerThread:75002 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-17 21:16:53,589 DEBUG   SenderThread:75002 [sender.py:send():379] send: files
+2024-12-17 21:16:53,589 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-17 21:16:53,589 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-17 21:16:53,589 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-17 21:16:53,817 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-17 21:16:53,817 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: python_packages
+2024-12-17 21:16:53,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:16:53,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:16:53,818 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:16:53,820 INFO    HandlerThread:75002 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/tb"
+  save: true
+}
+control {
+  relay_id: "eed105154fd946b4aa5434f527304c55"
+}
+_info {
+  stream_id: "14r6tvph"
+}
+
+2024-12-17 21:16:53,820 DEBUG   HandlerThread:75002 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-17 21:16:53,835 INFO    wandb-upload_2:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/0eiaj0fa-diff.patch
+2024-12-17 21:16:53,850 INFO    wandb-upload_0:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/j4lpzn7m-wandb-metadata.json
+2024-12-17 21:16:53,865 INFO    wandb-upload_1:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/puflo4o7-code/FISH/eval_robot.py
+2024-12-17 21:16:53,936 DEBUG   SenderThread:75002 [sender.py:send():379] send: telemetry
+2024-12-17 21:16:53,936 DEBUG   SenderThread:75002 [sender.py:send():379] send: tbrecord
+2024-12-17 21:16:53,936 DEBUG   SenderThread:75002 [sender.py:send():379] send: files
+2024-12-17 21:16:53,937 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0 with policy live
+2024-12-17 21:16:54,105 INFO    wandb-upload_2:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/8zd7zwie-events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0
+2024-12-17 21:16:54,402 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/diff.patch
+2024-12-17 21:16:54,403 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:16:54,403 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-metadata.json
+2024-12-17 21:16:54,403 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0
+2024-12-17 21:16:54,403 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/code/FISH/eval_robot.py
+2024-12-17 21:16:54,403 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/requirements.txt
+2024-12-17 21:16:54,403 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/code
+2024-12-17 21:16:54,403 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/code/FISH
+2024-12-17 21:16:54,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:16:55,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:16:56,402 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:16:56,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:16:57,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:16:58,660 DEBUG   SenderThread:75002 [sender.py:send():379] send: config
+2024-12-17 21:16:58,660 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:16:58,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:16:59,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:00,403 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:00,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:01,819 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:02,403 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:02,819 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:03,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:17:03,819 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:04,404 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:04,819 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:05,820 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:06,819 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:07,820 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:08,405 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:08,819 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:17:08,820 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:17:08,820 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:17:08,861 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:09,820 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:10,406 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:10,821 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:11,821 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:12,406 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:12,821 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:13,819 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:17:13,821 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:14,407 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:14,821 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:15,821 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:16,821 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:17,821 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:18,822 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:19,820 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:17:19,821 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:20,822 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:21,822 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:22,822 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:23,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:17:23,818 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:17:23,861 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:24,822 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:24,959 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:17:25,410 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/config.yaml
+2024-12-17 21:17:25,822 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:26,822 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:27,822 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:28,822 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:29,823 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:30,820 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:17:30,823 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:31,823 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:32,823 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:33,823 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:34,823 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:35,823 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:36,820 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:17:36,823 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:37,823 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:38,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:17:38,818 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:17:38,861 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:39,824 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:40,802 DEBUG   SenderThread:75002 [sender.py:send():379] send: files
+2024-12-17 21:17:40,802 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_607b63808c40640027f8.mp4 with policy now
+2024-12-17 21:17:40,803 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:17:40,803 DEBUG   SenderThread:75002 [sender.py:send():379] send: history
+2024-12-17 21:17:40,804 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:17:40,805 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:17:40,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:41,058 INFO    wandb-upload_1:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/htlcjn40-media/videos/eval/0_eval_0_607b63808c40640027f8.mp4
+2024-12-17 21:17:41,414 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json
+2024-12-17 21:17:41,414 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/0_eval_0_607b63808c40640027f8.mp4
+2024-12-17 21:17:41,414 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval
+2024-12-17 21:17:41,414 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos
+2024-12-17 21:17:41,414 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media
+2024-12-17 21:17:41,664 INFO    wandb-upload_2:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/7rdm6uyr-media/videos/eval/0_eval_0_607b63808c40640027f8.mp4
+2024-12-17 21:17:41,824 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:42,414 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:42,810 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:17:42,824 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:43,824 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:44,824 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:45,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:46,415 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:46,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:47,824 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:48,165 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:17:48,416 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:48,824 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:49,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:50,417 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0
+2024-12-17 21:17:50,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:51,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:52,418 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:52,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:53,232 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:17:53,494 DEBUG   SystemMonitor:75002 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-17 21:17:53,495 DEBUG   SenderThread:75002 [sender.py:send():379] send: stats
+2024-12-17 21:17:53,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:17:53,819 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:17:53,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:54,418 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:54,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:55,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:56,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:57,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:58,820 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:17:58,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:59,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:00,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:01,826 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:02,826 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:03,821 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:18:03,826 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:04,826 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:05,826 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:06,826 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:07,826 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:08,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:18:08,819 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:18:08,861 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:08,909 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:18:09,827 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:10,423 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:18:10,835 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:10,902 DEBUG   SenderThread:75002 [sender.py:send():379] send: files
+2024-12-17 21:18:10,903 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_528c8c53973fc263e4c2.mp4 with policy now
+2024-12-17 21:18:10,906 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:18:10,907 DEBUG   SenderThread:75002 [sender.py:send():379] send: history
+2024-12-17 21:18:10,907 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:18:10,907 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:18:11,136 INFO    wandb-upload_1:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/c3bv692d-media/videos/eval/1_eval_1_528c8c53973fc263e4c2.mp4
+2024-12-17 21:18:11,424 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json
+2024-12-17 21:18:11,424 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/1_eval_1_528c8c53973fc263e4c2.mp4
+2024-12-17 21:18:11,424 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval
+2024-12-17 21:18:11,658 INFO    wandb-upload_2:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/nduniaqo-media/videos/eval/1_eval_1_528c8c53973fc263e4c2.mp4
+2024-12-17 21:18:11,837 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:12,424 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:18:12,839 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:13,424 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:18:13,840 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:14,127 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:18:14,839 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:15,837 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:16,425 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:18:16,839 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:17,837 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:18,426 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:18:18,837 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:19,821 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:18:19,838 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:20,838 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:21,837 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:22,837 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:23,501 DEBUG   SenderThread:75002 [sender.py:send():379] send: stats
+2024-12-17 21:18:23,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:18:23,819 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:18:23,865 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:24,838 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:24,907 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:18:25,838 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:26,839 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:27,838 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:28,839 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:29,839 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:30,822 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:18:30,840 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:31,839 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:32,840 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:33,840 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:34,840 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:35,841 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:36,822 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:18:36,845 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:37,843 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:38,819 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:18:38,819 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:18:38,861 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:39,843 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:40,843 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:41,849 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:41,895 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:18:42,843 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:43,844 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:44,844 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:45,248 DEBUG   SenderThread:75002 [sender.py:send():379] send: history
+2024-12-17 21:18:45,248 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:18:45,249 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:18:45,435 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0
+2024-12-17 21:18:45,435 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json
+2024-12-17 21:18:45,660 INFO    wandb-upload_1:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/5dz2mi6h-events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0
+2024-12-17 21:18:45,844 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:46,435 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:18:46,844 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:47,649 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:18:47,844 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:48,436 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:18:48,844 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:49,844 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:50,437 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:18:50,844 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:51,844 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:52,823 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:18:52,845 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:53,496 DEBUG   SenderThread:75002 [sender.py:send():379] send: stats
+2024-12-17 21:18:53,819 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:18:53,819 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:18:53,861 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:54,846 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:55,846 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:56,846 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:57,846 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:57,946 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:18:58,846 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:59,846 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:00,440 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:00,863 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:01,859 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:02,189 DEBUG   SenderThread:75002 [sender.py:send():379] send: files
+2024-12-17 21:19:02,189 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_2f040d8fedc4c652052d.mp4 with policy now
+2024-12-17 21:19:02,191 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:19:02,192 DEBUG   SenderThread:75002 [sender.py:send():379] send: history
+2024-12-17 21:19:02,192 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:19:02,192 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:19:02,427 INFO    wandb-upload_2:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/2lqs5gyr-media/videos/eval/2_eval_2_2f040d8fedc4c652052d.mp4
+2024-12-17 21:19:02,440 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:02,441 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json
+2024-12-17 21:19:02,441 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/2_eval_2_2f040d8fedc4c652052d.mp4
+2024-12-17 21:19:02,441 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval
+2024-12-17 21:19:02,689 INFO    wandb-upload_1:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/d359f6i1-media/videos/eval/2_eval_2_2f040d8fedc4c652052d.mp4
+2024-12-17 21:19:02,855 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:03,208 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:19:03,441 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:03,860 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:04,441 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:04,860 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:05,861 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:06,862 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:07,860 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:08,442 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:08,462 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:19:08,819 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:19:08,819 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:19:08,859 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:09,861 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:10,443 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:10,863 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:11,863 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:12,863 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:13,828 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:19:13,866 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:14,863 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:15,866 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:16,867 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:17,880 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:18,829 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:19:18,879 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:19,883 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:20,047 DEBUG   SenderThread:75002 [sender.py:send():379] send: history
+2024-12-17 21:19:20,048 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:19:20,057 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:19:20,446 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0
+2024-12-17 21:19:20,446 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json
+2024-12-17 21:19:20,880 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:21,881 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:22,447 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:22,883 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:23,496 DEBUG   SenderThread:75002 [sender.py:send():379] send: stats
+2024-12-17 21:19:23,826 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:19:23,826 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:19:23,883 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:23,943 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:19:24,448 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:24,886 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:25,886 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:26,448 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:26,887 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:27,888 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:28,449 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:28,886 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:29,833 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:19:29,887 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:30,887 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:31,887 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:32,887 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:33,887 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:34,887 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:35,830 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:19:35,887 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:36,452 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:36,888 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:37,888 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:38,452 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:38,547 DEBUG   SenderThread:75002 [sender.py:send():379] send: files
+2024-12-17 21:19:38,547 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_d96484f372adc32d22ea.mp4 with policy now
+2024-12-17 21:19:38,549 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:19:38,550 DEBUG   SenderThread:75002 [sender.py:send():379] send: history
+2024-12-17 21:19:38,550 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:19:38,550 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:19:38,769 INFO    wandb-upload_2:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/gbinh9n5-media/videos/eval/3_eval_3_d96484f372adc32d22ea.mp4
+2024-12-17 21:19:38,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:19:38,825 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:19:38,900 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:39,453 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json
+2024-12-17 21:19:39,453 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/3_eval_3_d96484f372adc32d22ea.mp4
+2024-12-17 21:19:39,453 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval
+2024-12-17 21:19:39,831 INFO    wandb-upload_1:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/3yzpie16-media/videos/eval/3_eval_3_d96484f372adc32d22ea.mp4
+2024-12-17 21:19:39,899 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:40,453 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:40,893 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:40,942 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:19:41,896 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:42,901 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:43,906 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:44,454 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:44,901 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:45,905 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:45,953 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:19:46,455 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:46,905 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:47,917 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:48,924 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:49,905 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:50,913 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:51,839 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:19:51,907 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:52,911 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:53,497 DEBUG   SenderThread:75002 [sender.py:send():379] send: stats
+2024-12-17 21:19:53,828 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:19:53,828 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:19:53,913 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:54,921 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:55,087 DEBUG   SenderThread:75002 [sender.py:send():379] send: history
+2024-12-17 21:19:55,087 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:19:55,088 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:19:55,458 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0
+2024-12-17 21:19:55,458 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json
+2024-12-17 21:19:55,701 INFO    wandb-upload_2:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/4lo3xwe5-events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0
+2024-12-17 21:19:55,922 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:56,458 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:56,922 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:57,081 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:19:57,928 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:58,459 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:58,924 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:59,930 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:00,459 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:20:00,941 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:01,923 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:02,266 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:20:02,460 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:20:02,923 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:03,923 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:04,923 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:05,923 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:06,923 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:07,833 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:20:07,924 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:08,824 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:20:08,824 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:20:08,924 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:09,924 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:10,924 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:11,924 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:12,463 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:20:12,924 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:13,390 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:20:13,937 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:14,464 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:20:14,619 DEBUG   SenderThread:75002 [sender.py:send():379] send: files
+2024-12-17 21:20:14,619 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_85f8c5555fdc1daec6f7.mp4 with policy now
+2024-12-17 21:20:14,622 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:20:14,622 DEBUG   SenderThread:75002 [sender.py:send():379] send: history
+2024-12-17 21:20:14,622 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:20:14,622 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:20:14,849 INFO    wandb-upload_1:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/1gbzb2e6-media/videos/eval/4_eval_4_85f8c5555fdc1daec6f7.mp4
+2024-12-17 21:20:14,934 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:15,464 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json
+2024-12-17 21:20:15,465 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/4_eval_4_85f8c5555fdc1daec6f7.mp4
+2024-12-17 21:20:15,465 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval
+2024-12-17 21:20:15,729 INFO    wandb-upload_2:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/194n5l0n-media/videos/eval/4_eval_4_85f8c5555fdc1daec6f7.mp4
+2024-12-17 21:20:15,933 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:16,465 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:20:16,937 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:17,935 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:18,836 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:20:18,963 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:19,936 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:20,466 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:20:20,939 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:21,950 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:22,467 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:20:22,942 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:23,497 DEBUG   SenderThread:75002 [sender.py:send():379] send: stats
+2024-12-17 21:20:23,837 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:20:23,837 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:20:23,950 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:23,954 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:20:24,938 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:25,949 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:26,977 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:27,948 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:28,955 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:29,850 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:20:29,961 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:30,952 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:31,955 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:32,953 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:33,956 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:34,957 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:35,121 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:20:35,131 DEBUG   SenderThread:75002 [sender.py:send():379] send: history
+2024-12-17 21:20:35,131 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:20:35,132 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:20:35,165 DEBUG   SenderThread:75002 [sender.py:send():379] send: exit
+2024-12-17 21:20:35,165 INFO    SenderThread:75002 [sender.py:send_exit():586] handling exit code: 0
+2024-12-17 21:20:35,166 INFO    SenderThread:75002 [sender.py:send_exit():588] handling runtime: 221
+2024-12-17 21:20:35,166 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:20:35,166 INFO    SenderThread:75002 [sender.py:send_exit():594] send defer
+2024-12-17 21:20:35,166 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:35,166 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-17 21:20:35,166 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:35,166 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-17 21:20:35,166 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 1
+2024-12-17 21:20:35,166 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:35,166 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-17 21:20:35,167 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:35,167 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-17 21:20:35,167 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 2
+2024-12-17 21:20:35,167 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:35,167 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-17 21:20:35,167 INFO    HandlerThread:75002 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-17 21:20:35,167 DEBUG   SystemMonitor:75002 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-17 21:20:35,167 DEBUG   SystemMonitor:75002 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-17 21:20:35,167 INFO    HandlerThread:75002 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-17 21:20:35,168 INFO    HandlerThread:75002 [interfaces.py:finish():200] Joined disk monitor
+2024-12-17 21:20:35,217 INFO    HandlerThread:75002 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-17 21:20:35,218 INFO    HandlerThread:75002 [interfaces.py:finish():200] Joined memory monitor
+2024-12-17 21:20:35,218 INFO    HandlerThread:75002 [interfaces.py:finish():200] Joined network monitor
+2024-12-17 21:20:35,218 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:35,218 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-17 21:20:35,218 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 3
+2024-12-17 21:20:35,218 DEBUG   SenderThread:75002 [sender.py:send():379] send: stats
+2024-12-17 21:20:35,219 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:35,219 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-17 21:20:35,219 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:35,219 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-17 21:20:35,219 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 4
+2024-12-17 21:20:35,219 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:35,219 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-17 21:20:35,471 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0
+2024-12-17 21:20:35,471 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json
+2024-12-17 21:20:36,471 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:20:41,220 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:20:41,220 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:20:41,220 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:41,221 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-17 21:20:41,221 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-17 21:20:41,221 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 5
+2024-12-17 21:20:41,221 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:20:41,221 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:41,221 DEBUG   SenderThread:75002 [sender.py:send():379] send: history
+2024-12-17 21:20:41,221 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-17 21:20:41,222 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:20:41,222 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:20:41,222 DEBUG   SenderThread:75002 [sender.py:send():379] send: summary
+2024-12-17 21:20:41,226 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:20:41,226 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:41,226 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-17 21:20:41,226 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 6
+2024-12-17 21:20:41,226 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:41,226 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-17 21:20:41,226 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:41,227 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-17 21:20:41,227 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 7
+2024-12-17 21:20:41,227 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:20:41,227 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:41,227 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-17 21:20:41,227 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:41,227 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-17 21:20:41,473 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json
+2024-12-17 21:20:42,166 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:20:42,831 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 8
+2024-12-17 21:20:42,831 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:20:42,831 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:42,832 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-17 21:20:42,832 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:42,832 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-17 21:20:42,832 INFO    SenderThread:75002 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-17 21:20:42,834 INFO    SenderThread:75002 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-17 21:20:42,849 INFO    SenderThread:75002 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-17 21:20:42,859 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 9
+2024-12-17 21:20:42,859 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:42,859 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-17 21:20:42,860 DEBUG   SenderThread:75002 [sender.py:send():379] send: artifact
+2024-12-17 21:20:43,167 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:20:43,275 INFO    wandb-upload_0:75002 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp58wukwcy
+2024-12-17 21:20:43,276 INFO    wandb-upload_2:75002 [upload_job.py:push():85] Skipped uploading /tmp/tmpyn9jqz0t/wandb-job.json
+2024-12-17 21:20:43,474 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:20:43,479 INFO    wandb-upload_1:75002 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpvalkza0y
+2024-12-17 21:20:44,190 INFO    SenderThread:75002 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM4MzE5ODcyNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM4MzE0NTc2Mg==', 'versionIndex': 377}}}
+2024-12-17 21:20:44,190 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:44,190 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-17 21:20:44,190 INFO    SenderThread:75002 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-17 21:20:44,475 INFO    SenderThread:75002 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files
+2024-12-17 21:20:44,475 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/config.yaml config.yaml
+2024-12-17 21:20:44,475 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0 events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0
+2024-12-17 21:20:44,477 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json wandb-summary.json
+2024-12-17 21:20:44,477 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/requirements.txt requirements.txt
+2024-12-17 21:20:44,477 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-metadata.json wandb-metadata.json
+2024-12-17 21:20:44,477 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log output.log
+2024-12-17 21:20:44,477 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/diff.patch diff.patch
+2024-12-17 21:20:44,477 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-17 21:20:44,483 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/3_eval_3_d96484f372adc32d22ea.mp4 media/videos/eval/3_eval_3_d96484f372adc32d22ea.mp4
+2024-12-17 21:20:44,483 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/0_eval_0_607b63808c40640027f8.mp4 media/videos/eval/0_eval_0_607b63808c40640027f8.mp4
+2024-12-17 21:20:44,484 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/1_eval_1_528c8c53973fc263e4c2.mp4 media/videos/eval/1_eval_1_528c8c53973fc263e4c2.mp4
+2024-12-17 21:20:44,484 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/2_eval_2_2f040d8fedc4c652052d.mp4 media/videos/eval/2_eval_2_2f040d8fedc4c652052d.mp4
+2024-12-17 21:20:44,484 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/4_eval_4_85f8c5555fdc1daec6f7.mp4 media/videos/eval/4_eval_4_85f8c5555fdc1daec6f7.mp4
+2024-12-17 21:20:44,484 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 10
+2024-12-17 21:20:44,484 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:20:44,484 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:44,484 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-17 21:20:44,484 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:44,484 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-17 21:20:44,484 INFO    SenderThread:75002 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 21:20:44,647 INFO    wandb-upload_3:75002 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json
+2024-12-17 21:20:44,653 INFO    wandb-upload_0:75002 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/config.yaml
+2024-12-17 21:20:44,691 INFO    wandb-upload_4:75002 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:20:44,695 INFO    wandb-upload_2:75002 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/requirements.txt
+2024-12-17 21:20:44,698 INFO    wandb-upload_1:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/eajpd6gl-events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0
+2024-12-17 21:20:44,898 INFO    Thread-11 (_thread_body):75002 [sender.py:transition_state():614] send defer: 11
+2024-12-17 21:20:44,899 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:44,899 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-17 21:20:44,899 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:44,899 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-17 21:20:44,899 INFO    SenderThread:75002 [file_pusher.py:join():175] waiting for file pusher
+2024-12-17 21:20:44,900 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 12
+2024-12-17 21:20:44,900 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:44,900 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-17 21:20:44,900 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:44,900 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-17 21:20:44,900 INFO    SenderThread:75002 [file_stream.py:finish():601] file stream finish called
+2024-12-17 21:20:44,962 INFO    SenderThread:75002 [file_stream.py:finish():605] file stream finish is done
+2024-12-17 21:20:44,962 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 13
+2024-12-17 21:20:44,962 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:44,962 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-17 21:20:44,962 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:44,962 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-17 21:20:44,962 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 14
+2024-12-17 21:20:44,962 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:44,963 DEBUG   SenderThread:75002 [sender.py:send():379] send: final
+2024-12-17 21:20:44,963 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-17 21:20:44,963 DEBUG   SenderThread:75002 [sender.py:send():379] send: footer
+2024-12-17 21:20:44,963 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:44,963 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-17 21:20:44,963 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:20:44,963 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:20:44,964 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: server_info
+2024-12-17 21:20:44,964 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: server_info
+2024-12-17 21:20:44,965 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:20:44,965 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-17 21:20:44,966 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-17 21:20:44,966 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:45,040 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:20:45,040 INFO    MainThread:75002 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-17 21:20:45,041 INFO    MainThread:75002 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-17 21:20:45,041 INFO    MainThread:75002 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-17 21:20:45,041 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-17 21:20:45,041 INFO    HandlerThread:75002 [handler.py:finish():882] shutting down handler
+2024-12-17 21:20:45,966 INFO    WriterThread:75002 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/run-14r6tvph.wandb
+2024-12-17 21:20:46,040 INFO    SenderThread:75002 [sender.py:finish():1615] shutting down sender
+2024-12-17 21:20:46,041 INFO    SenderThread:75002 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 21:20:46,041 INFO    SenderThread:75002 [file_pusher.py:join():175] waiting for file pusher

211647/wandb/debug.log

@@ -0,0 +1,31 @@
+2024-12-17 21:16:53,162 INFO    MainThread:74816 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-17 21:16:53,162 INFO    MainThread:74816 [wandb_setup.py:_flush():76] Configure stats pid to 74816
+2024-12-17 21:16:53,162 INFO    MainThread:74816 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-17 21:16:53,162 INFO    MainThread:74816 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/settings
+2024-12-17 21:16:53,162 INFO    MainThread:74816 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-17 21:16:53,162 INFO    MainThread:74816 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-17 21:16:53,162 INFO    MainThread:74816 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-17 21:16:53,162 INFO    MainThread:74816 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-17 21:16:53,162 INFO    MainThread:74816 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/logs/debug.log
+2024-12-17 21:16:53,162 INFO    MainThread:74816 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/logs/debug-internal.log
+2024-12-17 21:16:53,163 INFO    MainThread:74816 [wandb_init.py:init():569] calling init triggers
+2024-12-17 21:16:53,164 INFO    MainThread:74816 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '508_0', 'wandb_notes': '508_0_req_579_0', 'eval': True, 'true_action_history': False, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': False, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['fowlers_twodim'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/508_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647'}
+2024-12-17 21:16:53,164 INFO    MainThread:74816 [wandb_init.py:init():619] starting backend
+2024-12-17 21:16:53,164 INFO    MainThread:74816 [wandb_init.py:init():623] setting up manager
+2024-12-17 21:16:53,170 INFO    MainThread:74816 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-17 21:16:53,172 INFO    MainThread:74816 [wandb_init.py:init():631] backend started and connected
+2024-12-17 21:16:53,182 INFO    MainThread:74816 [wandb_init.py:init():720] updated telemetry
+2024-12-17 21:16:53,189 INFO    MainThread:74816 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-17 21:16:53,413 INFO    MainThread:74816 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-17 21:16:53,482 INFO    MainThread:74816 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-17 21:16:53,482 INFO    MainThread:74816 [wandb_init.py:init():804] starting run threads in backend
+2024-12-17 21:16:53,817 INFO    MainThread:74816 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-17 21:16:53,818 INFO    MainThread:74816 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-17 21:16:53,818 INFO    MainThread:74816 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-17 21:16:53,818 INFO    MainThread:74816 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-17 21:16:53,819 INFO    MainThread:74816 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-17 21:16:53,819 INFO    MainThread:74816 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/tb, True
+2024-12-17 21:16:58,659 INFO    MainThread:74816 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'}
+2024-12-17 21:20:46,044 WARNING MsgRouterThr:74816 [router.py:message_loop():77] message_loop has been closed

211647/wandb/run-20241217_211653-14r6tvph/files/code/FISH/eval_robot.py

@@ -0,0 +1,512 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)		
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()

211647/wandb/run-20241217_211653-14r6tvph/files/config.yaml

@@ -0,0 +1,891 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 0
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '508_0'
+wandb_notes:
+  desc: null
+  value: 508_0_req_579_0
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: false
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: false
+use_sdf_maps:
+  desc: null
+  value: false
+use_normals_maps:
+  desc: null
+  value: false
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value:
+  - fowlers_twodim
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 0
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/kostas-graid/datasets/extrinsic_contact_data/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/508_0
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: false
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: false
+        use_sdf_maps: false
+        use_normals_maps: false
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 0
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: false
+          use_sdf_maps: false
+          use_normals_maps: false
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1734488213
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 16
+      - 23
+      - 35
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: fowlers
+left_book_slot:
+  desc: null
+  value: twodim

211647/wandb/run-20241217_211653-14r6tvph/files/diff.patch

@@ -0,0 +1,195 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index bb0a413..de3612d 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -107,7 +107,7 @@ num_valid_demos: null
+ val_num_groups: 3
+ 
+ # name_of_expert_demo: 'book_insertion_wo_ends_wedge_v2_76_all_demos_10hz_no_filter_wo_flow_mask_expert_demos_imp_act'
+-name_of_expert_demo: '192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act'
++name_of_expert_demo: '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act'
+ expert_dataset_dirpath: '${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}'
+ 
+ expert_dataset: '${expert_dataset_dirpath}/demos.zarr'
+@@ -123,89 +123,28 @@ bc_weight_type: 'qfilter' # linear, qfilter
+ 
+ # Load weights
+ load_checkpoint: ${agent.load_checkpoint}
+-# 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# full 32 demos
+-# wandb_run_id: "265625" # RGBD history seed 0
+-# wandb_run_id: "266370_1" # RGBD history seed 1
+-# wandb_run_id: "266370_2" # RGBD history seed 2
+-# wandb_run_id: "266382_4" # RGBD history seed 4
+-
+-# 16 held out demos (16 demos)
+-# wandb_run_id: "266392_0" # RGBD+act history dataset seed 0
+-# wandb_run_id: "266407_0" # RGBD+act history dataset seed 1
+-wandb_run_id: "266406_0" # RGBD+act history dataset seed 2
+-
+-# wandb_run_id: "265626" # RGBD+Mask+act history seed 0
+-# wandb_run_id: "265638" # RGBD+Mask+act history seed 1
+-# wandb_run_id: "265637" # RGBD+Mask+act history seed 2
+-# wandb_run_id: "265636" # RGBD+Mask+act history seed 3
+-# wandb_run_id: "265635" # RGBD+Mask+act history seed 4
+-
+-# wandb_run_id: "265628" # RGBD+Mask+contact+act history seed 0
+-# wandb_run_id: "265630" # RGBD+Mask+contact+act history seed 1
+-# wandb_run_id: "265631" # RGBD+Mask+contact+act history seed 2
+-# wandb_run_id: "265633" # RGBD+Mask+contact+act history seed 3
+-# wandb_run_id: "265634" # RGBD+Mask+contact+act history seed 4
+-
+-# 8 held out demos (26 demos)
+-# wandb_run_id: "265652" # RGBD+Mask+contact+act history dataset seed 0
+-# wandb_run_id: "265654" # RGBD+Mask+contact+act history dataset seed 1
+-# wandb_run_id: "265655" # RGBD+Mask+contact+act history dataset seed 2
+-
+-# 8 held out demos (26 demos)
+-# wandb_run_id: "265656" # RGBD+Mask+act history dataset seed 0
+-# wandb_run_id: "265657" # RGBD+Mask+act history dataset seed 1
+-# wandb_run_id: "265658" # RGBD+Mask+act history dataset seed 2
+-
+-# 16 held out demos (16 demos)
+-# wandb_run_id: "265641" # RGBD+Mask+act history dataset seed 0
+-# wandb_run_id: "265644" # RGBD+Mask+act history dataset seed 1
+-# wandb_run_id: "265645" # RGBD+Mask+act history dataset seed 2
+-# wandb_run_id: "266381_0" # RGBD+Mask+act history dataset seed 4
+-
+-# 16 held out demos (16 demos)
+-# wandb_run_id: "265646" # RGBD+Mask+Contact+act history dataset seed 0
+-# wandb_run_id: "265649" # RGBD+Mask+Contact+act history dataset seed 1
+-# wandb_run_id: "265650" # RGBD+Mask+Contact+act history dataset seed 2
+-# wandb_run_id: "266378_0" # RGBD+Mask+Contact+act history dataset seed 4
+-
+-
+-
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# full dataset
+-# 6 step action history
+-# wandb_run_id: '250358_0' # RGBD+act history
+-# wandb_run_id: '250371_0' # RGBD+mask+act history
+-# wandb_run_id: '250366_0' # RGBD+contact+act history
+-# wandb_run_id: '250376_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250366_0' # RGBD+contact+act history
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# 16 held out demos
+-# wandb_run_id: '250362_0' # RGBD+act history
+-# wandb_run_id: '250373_0' # RGBD+mask+act history
+-# wandb_run_id: '250378_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250368_0' # RGBD+contact+act history
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# 32 held out demos
+-# wandb_run_id: '250365_0' # RGBD+act history
+-# wandb_run_id: '250375_0' # RGBD+mask+act history
+-wandb_run_id: '250380_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250370_0' # RGBD+contact+act history
+-# 4 step action history
+-# wandb_run_id: '264772_0' # RGBD+act history
+-# wandb_run_id: '264775_0' # RGBD+mask+act history
+-# wandb_run_id: '264776_0' # RGBD+contact+act history
+-# wandb_run_id: '264777_0' # RGBD+mask+contact+act history
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# 16 held out demos
+-# wandb_run_id: '250362_0' # RGBD+act history
+-# wandb_run_id: '250373_0' # RGBD+mask+act history
+-# wandb_run_id: '250378_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250368_0' # RGBD+contact+act history
++# 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
++# 14 demos
++
++# RGBD+act history 
++# wandb_run_id: '497_0' # german
++# wandb_run_id: '505_0' # hbm
++# wandb_run_id: '506_0' # lady
++# wandb_run_id: '507_0' # greece
++wandb_run_id: '508_0' # fowlers
++# wandb_run_id: '509_0' # lib
++# wandb_run_id: '510_0' # modelsys
++# wandb_run_id: '511_0' # electrodyn
++
++# RGBD+mask+act history
++# wandb_run_id: '512_0' # german
++# wandb_run_id: '520_0' # hbm
++# wandb_run_id: '521_0' # lady
++# wandb_run_id: '522_0' # greece
++# wandb_run_id: '523_0' # fowlers
++# wandb_run_id: '524_0' # lib
++# wandb_run_id: '525_0' # modelsys
++# wandb_run_id: '526_0' # electrodyn
+ 
+ # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # 32 held out demos
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index 9406a46..0f225ec 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -15,12 +15,24 @@ if not local_checkpoint_root_dir.exists():
+     local_checkpoint_root_dir.mkdir(parents=True)
+ 
+ # Download the model checkpoints
+-remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1')
++# remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1')
++remote_checkpoint_root_dir = Path('/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1')
+ remote_checkpoint_root_dir = remote_checkpoint_root_dir.expanduser()
+ remote_username = 'leonmkim'
+-remote_host = 'halley'
++remote_host = 'grasp-login1'
+ 
+ #180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
++# run_id_list = [
++#     '512_0', # german
++#     '520_0', # hbm
++#     '521_0', # lady
++#     '522_0', # greece
++#     '523_0', # fowlers
++#     '524_0', # lib
++#     '525_0', # modelsys
++#     '526_0', # electrodyn
++# ]
++
+ # run_id_list = [
+ #     '265638',
+ #     '265637',
+@@ -75,6 +87,17 @@ remote_host = 'halley'
+ # ]
+ 
+ # 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
++# run_id_list = [
++#     '497_0', # german
++#     '505_0', # hbm
++#     '506_0', # lady
++#     '507_0', # greece
++#     '508_0', # fowlers
++#     '509_0', # lib
++#     '510_0', # modelsys
++#     '511_0', # electrodyn
++# ]
++
+ # run_id_list = [
+ #     '266382_4',
+ #     '266370_2',
+@@ -116,13 +139,13 @@ remote_host = 'halley'
+ # ]
+ 
+ # 180x240_1_RGB_D_2.0_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+-run_id_list = [
+-    # '266384_4,# not yet available
+-    '266372_1',
+-    '266372_2',
+-    '265627',
+-    '265620',
+-]
++# run_id_list = [
++#     # '266384_4,# not yet available
++#     '266372_1',
++#     '266372_2',
++#     '265627',
++#     '265620',
++# ]
+ checkpoint_epoch = 12000
+ 
+ # use subprocess to download the model checkpoints in parallel

211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/0_eval_0_607b63808c40640027f8.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:607b63808c40640027f8da1ae9902a7aa7037aa7b73881ae52fcd560abc6131c
+size 1162092

211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/1_eval_1_528c8c53973fc263e4c2.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:528c8c53973fc263e4c239f72cdd4f717b7ebe1bafb47099a337bf475618128c
+size 734029

211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/2_eval_2_2f040d8fedc4c652052d.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f040d8fedc4c652052df87e67a190bae28ff42773fdd3024edaf607ac6a0bd0
+size 585337

211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/3_eval_3_d96484f372adc32d22ea.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d96484f372adc32d22ea328440dc6fca791adad01df8190f1167d57835cf3588
+size 640584

211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/4_eval_4_85f8c5555fdc1daec6f7.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85f8c5555fdc1daec6f75870c681078310d2198c2dbcf62e2cdc8271016d199f
+size 670569

211647/wandb/run-20241217_211653-14r6tvph/files/output.log

@@ -0,0 +1,231 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1734488218.660370]: resetting environment
+[INFO] [1734488218.662326]: cleared current plan
+[INFO] [1734488218.663923]: moving to home
+[INFO] [1734488219.667458]: reached home
+[INFO] [1734488219.669374]: reset action history
+[INFO] [1734488222.376990]: environment reset
+Starting episode 0
+[INFO] [1734488222.379328]: resetting environment
+[INFO] [1734488222.380062]: cleared current plan
+[INFO] [1734488222.380895]: moving to home
+[INFO] [1734488223.382916]: reached home
+[INFO] [1734488223.384527]: reset action history
+[INFO] [1734488226.091900]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1734488229.363950]: resetting environment
+[INFO] [1734488229.365239]: cleared current plan
+[INFO] [1734488229.366266]: moving to home
+[INFO] [1734488230.368748]: reached home
+[INFO] [1734488230.369981]: reset action history
+[INFO] [1734488233.076918]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/episode_rosbags/episode_0_2024-12-17-21-17-13.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-74816.log
+started roslaunch server http://158.130.50.37:32957/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [75102]
+process[print_text-2]: started with pid [75126]
+process[rosbag_record-3]: started with pid [75127]
+started bagging!
+[INFO] [1734488259.862923]: Storing episode...
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit[pub_text-1] killing on exit
+[INFO] [1734488260.586914]: Stored episode 1.
+[INFO] [1734488260.587408]: Saving video...
+[INFO] [1734488260.791339]: Video saved!
+Starting episode 1
+[INFO] [1734488260.803108]: resetting environment
+[INFO] [1734488260.803717]: cleared current plan
+[INFO] [1734488260.806602]: moving to home
+[INFO] [1734488264.413284]: reached home
+[INFO] [1734488264.414431]: reset action history
+[INFO] [1734488267.120749]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1734488269.523343]: resetting environment
+[INFO] [1734488269.523522]: cleared current plan
+[INFO] [1734488269.523891]: moving to home
+[INFO] [1734488270.525244]: reached home
+[INFO] [1734488270.525734]: reset action history
+[INFO] [1734488273.231854]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/episode_rosbags/episode_1_2024-12-17-21-17-53.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-74816.log
+started roslaunch server http://158.130.50.37:40659/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [75240]
+process[print_text-5]: started with pid [75241]
+process[rosbag_record-6]: started with pid [75242]
+started bagging!
+preempting episode
+[INFO] [1734488289.446437]: Storing episode...
+[rosbag_record-6] killing on exit
+[print_text-5] killing on exit
+[pub_text-4] killing on exit
+[INFO] [1734488290.223045]: Stored episode 2.
+[INFO] [1734488290.223777]: Saving video...
+[INFO] [1734488290.863283]: Video saved!
+Starting episode 2
+[INFO] [1734488290.911673]: resetting environment
+[INFO] [1734488290.915867]: cleared current plan
+[INFO] [1734488290.919450]: moving to home
+[WARN] [1734488290.693148]: Plan exhausted
+[WARN] [1734488290.733547]: Plan exhausted
+[WARN] [1734488290.772900]: Plan exhausted
+[WARN] [1734488290.812064]: Plan exhausted
+[WARN] [1734488290.852860]: Plan exhausted
+[WARN] [1734488290.891611]: Plan exhausted
+[INFO] [1734488294.126837]: reached home
+[INFO] [1734488294.131333]: reset action history
+[INFO] [1734488296.873024]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode![INFO] [1734488324.594563]: resetting environment
+[INFO] [1734488324.595213]: cleared current plan
+[INFO] [1734488324.599433]: moving to home
+[INFO] [1734488325.606330]: reached home
+[INFO] [1734488325.609797]: reset action history
+[INFO] [1734488328.318138]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/episode_rosbags/episode_2_2024-12-17-21-18-48.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-74816.log
+started roslaunch server http://158.130.50.37:36773/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [75396]
+process[print_text-8]: started with pid [75397]
+process[rosbag_record-9]: started with pid [75398]
+started bagging!
+For topic gripper_width: timestamp difference is -127923472 for nearest: 1734488338919779079 - target: 1734488339047702551 at idx 148
+preempting episode
+[INFO] [1734488340.713237]: Storing episode...
+[rosbag_record-9] killing on exit
+[print_text-8] killing on exit
+[pub_text-7] killing on exit
+[INFO] [1734488341.524542]: Stored episode 3.
+[INFO] [1734488341.525559]: Saving video...
+[WARN] [1734488342.051578]: Plan exhausted
+[WARN] [1734488342.091628]: Plan exhausted
+[WARN] [1734488342.131796]: Plan exhausted
+[WARN] [1734488342.171625]: Plan exhausted
+[INFO] [1734488342.157426]: Video saved!
+Starting episode 3
+[INFO] [1734488342.196753]: resetting environment
+[INFO] [1734488342.205289]: cleared current plan
+[INFO] [1734488342.208267]: moving to home
+[INFO] [1734488345.722708]: reached home
+[INFO] [1734488345.726239]: reset action history
+[INFO] [1734488348.457332]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode![INFO] [1734488359.764443]: resetting environment
+[INFO] [1734488359.776920]: cleared current plan
+[INFO] [1734488359.780439]: moving to home
+proceeding to start episode!
+[INFO] [1734488360.783718]: reached home
+[INFO] [1734488360.787811]: reset action history
+[INFO] [1734488363.562521]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/episode_rosbags/episode_3_2024-12-17-21-19-23.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-74816.log
+started roslaunch server http://158.130.50.37:41431/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-10]: started with pid [75523]
+process[print_text-11]: started with pid [75524]
+process[rosbag_record-12]: started with pid [75525]
+started bagging!
+For topic gripper_width: timestamp difference is -106247271 for nearest: 1734488366519780790 - target: 1734488366626028061 at idx 45
+For topic gripper_width: timestamp difference is -109525155 for nearest: 1734488373519781638 - target: 1734488373629306793 at idx 139
+preempting episode
+[INFO] [1734488377.440101]: Storing episode...
+[rosbag_record-12] killing on exit[print_text-11] killing on exit
+[pub_text-10] killing on exit
+[INFO] [1734488378.276652]: Stored episode 4.
+[INFO] [1734488378.279107]: Saving video...
+[INFO] [1734488378.498791]: Video saved!
+Starting episode 4
+[INFO] [1734488378.555450]: resetting environment
+[INFO] [1734488378.558624]: cleared current plan
+[INFO] [1734488378.570886]: moving to home
+[INFO] [1734488382.098369]: reached home
+[INFO] [1734488382.101175]: reset action history
+[INFO] [1734488384.910976]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode![INFO] [1734488395.012886]: resetting environment
+[INFO] [1734488395.027065]: cleared current plan
+[INFO] [1734488395.028082]: moving to home
+[INFO] [1734488396.032406]: reached home
+[INFO] [1734488396.045183]: reset action history
+[INFO] [1734488398.848399]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/episode_rosbags/episode_4_2024-12-17-21-19-58.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-74816.log
+started roslaunch server http://158.130.50.37:41943/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-13]: started with pid [75650]
+process[print_text-14]: started with pid [75651]
+process[rosbag_record-15]: started with pid [75652]
+started bagging!
+For topic gripper_width: timestamp difference is -102728972 for nearest: 1734488411419777265 - target: 1734488411522506237 at idx 157
+For topic gripper_width: timestamp difference is -101248113 for nearest: 1734488413219801577 - target: 1734488413321049690 at idx 175
+preempting episode
+[INFO] [1734488413.539313]: Storing episode...
+[rosbag_record-15] killing on exit
+[pub_text-13] killing on exit
+[print_text-14] killing on exit
+[INFO] [1734488414.357617]: Stored episode 5.
+[INFO] [1734488414.365719]: Saving video...
+[INFO] [1734488414.561425]: Video saved!
+[INFO] [1734488414.622346]: resetting environment
+[INFO] [1734488414.628233]: cleared current plan
+[INFO] [1734488414.629943]: moving to home
+[INFO] [1734488418.143390]: reached home
+[INFO] [1734488418.150385]: reset action history
+[INFO] [1734488421.006638]: environment reset
+Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!

211647/wandb/run-20241217_211653-14r6tvph/files/requirements.txt

@@ -0,0 +1,339 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.2
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0

211647/wandb/run-20241217_211653-14r6tvph/files/wandb-metadata.json

@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2024-12-18T02:16:53.503865",
+    "startedAt": "2024-12-18T02:16:53.161156",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "5f28b54373d7d41e4ebace380ff1080a8ffc4051"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 3747.0996250000003,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4495.328,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.364,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 386.0898132324219
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}

211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json

@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "607b63808c40640027f8da1ae9902a7aa7037aa7b73881ae52fcd560abc6131c", "size": 1162092, "path": "media/videos/eval/0_eval_0_607b63808c40640027f8.mp4"}, "global_step": 386, "_timestamp": 1734488435.1219776, "_runtime": 221.9495005607605, "_step": 9, "eval/1_eval": {"_type": "video-file", "sha256": "528c8c53973fc263e4c239f72cdd4f717b7ebe1bafb47099a337bf475618128c", "size": 734029, "path": "media/videos/eval/1_eval_1_528c8c53973fc263e4c2.mp4"}, "eval/num_success": 1.0, "episode": 5.0, "eval/success_rate": 0.20000000298023224, "eval/2_eval": {"_type": "video-file", "sha256": "2f040d8fedc4c652052df87e67a190bae28ff42773fdd3024edaf607ac6a0bd0", "size": 585337, "path": "media/videos/eval/2_eval_2_2f040d8fedc4c652052d.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "d96484f372adc32d22ea328440dc6fca791adad01df8190f1167d57835cf3588", "size": 640584, "path": "media/videos/eval/3_eval_3_d96484f372adc32d22ea.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "85f8c5555fdc1daec6f75870c681078310d2198c2dbcf62e2cdc8271016d199f", "size": 670569, "path": "media/videos/eval/4_eval_4_85f8c5555fdc1daec6f7.mp4"}, "_wandb": {"runtime": 221}}

211647/wandb/run-20241217_211653-14r6tvph/logs/debug-internal.log

@@ -0,0 +1,650 @@
+2024-12-17 21:16:53,171 INFO    StreamThr :75002 [internal.py:wandb_internal():85] W&B internal server running at pid: 75002, started at: 2024-12-17 21:16:53.170951
+2024-12-17 21:16:53,172 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status
+2024-12-17 21:16:53,183 INFO    WriterThread:75002 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/run-14r6tvph.wandb
+2024-12-17 21:16:53,183 DEBUG   SenderThread:75002 [sender.py:send():379] send: header
+2024-12-17 21:16:53,191 DEBUG   SenderThread:75002 [sender.py:send():379] send: run
+2024-12-17 21:16:53,401 INFO    SenderThread:75002 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files
+2024-12-17 21:16:53,401 INFO    SenderThread:75002 [sender.py:_start_run_threads():1188] run started: 14r6tvph with start time 1734488213.172477
+2024-12-17 21:16:53,414 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: check_version
+2024-12-17 21:16:53,414 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: check_version
+2024-12-17 21:16:53,486 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: run_start
+2024-12-17 21:16:53,491 DEBUG   HandlerThread:75002 [system_info.py:__init__():26] System info init
+2024-12-17 21:16:53,491 DEBUG   HandlerThread:75002 [system_info.py:__init__():41] System info init done
+2024-12-17 21:16:53,491 INFO    HandlerThread:75002 [system_monitor.py:start():194] Starting system monitor
+2024-12-17 21:16:53,491 INFO    SystemMonitor:75002 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-17 21:16:53,491 INFO    HandlerThread:75002 [system_monitor.py:probe():214] Collecting system info
+2024-12-17 21:16:53,492 INFO    SystemMonitor:75002 [interfaces.py:start():188] Started cpu monitoring
+2024-12-17 21:16:53,492 INFO    SystemMonitor:75002 [interfaces.py:start():188] Started disk monitoring
+2024-12-17 21:16:53,493 INFO    SystemMonitor:75002 [interfaces.py:start():188] Started gpu monitoring
+2024-12-17 21:16:53,493 INFO    SystemMonitor:75002 [interfaces.py:start():188] Started memory monitoring
+2024-12-17 21:16:53,494 INFO    SystemMonitor:75002 [interfaces.py:start():188] Started network monitoring
+2024-12-17 21:16:53,503 DEBUG   HandlerThread:75002 [system_info.py:probe():152] Probing system
+2024-12-17 21:16:53,505 DEBUG   HandlerThread:75002 [system_info.py:_probe_git():137] Probing git
+2024-12-17 21:16:53,510 DEBUG   HandlerThread:75002 [system_info.py:_probe_git():145] Probing git done
+2024-12-17 21:16:53,510 DEBUG   HandlerThread:75002 [system_info.py:probe():200] Probing system done
+2024-12-17 21:16:53,511 DEBUG   HandlerThread:75002 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-18T02:16:53.503865', 'startedAt': '2024-12-18T02:16:53.161156', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '5f28b54373d7d41e4ebace380ff1080a8ffc4051'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3747.0996250000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4495.328, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.364, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 386.0898132324219}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2024-12-17 21:16:53,511 INFO    HandlerThread:75002 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-17 21:16:53,511 INFO    HandlerThread:75002 [system_monitor.py:probe():227] Publishing system info
+2024-12-17 21:16:53,511 DEBUG   HandlerThread:75002 [system_info.py:_save_code():44] Saving code
+2024-12-17 21:16:53,512 DEBUG   HandlerThread:75002 [system_info.py:_save_code():67] Saving code done
+2024-12-17 21:16:53,512 DEBUG   HandlerThread:75002 [system_info.py:_save_patches():84] Saving git patches
+2024-12-17 21:16:53,585 DEBUG   HandlerThread:75002 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-17 21:16:53,586 INFO    HandlerThread:75002 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-17 21:16:53,589 DEBUG   SenderThread:75002 [sender.py:send():379] send: files
+2024-12-17 21:16:53,589 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-17 21:16:53,589 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-17 21:16:53,589 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-17 21:16:53,817 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-17 21:16:53,817 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: python_packages
+2024-12-17 21:16:53,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:16:53,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:16:53,818 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:16:53,820 INFO    HandlerThread:75002 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/tb"
+  save: true
+}
+control {
+  relay_id: "eed105154fd946b4aa5434f527304c55"
+}
+_info {
+  stream_id: "14r6tvph"
+}
+
+2024-12-17 21:16:53,820 DEBUG   HandlerThread:75002 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-17 21:16:53,835 INFO    wandb-upload_2:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/0eiaj0fa-diff.patch
+2024-12-17 21:16:53,850 INFO    wandb-upload_0:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/j4lpzn7m-wandb-metadata.json
+2024-12-17 21:16:53,865 INFO    wandb-upload_1:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/puflo4o7-code/FISH/eval_robot.py
+2024-12-17 21:16:53,936 DEBUG   SenderThread:75002 [sender.py:send():379] send: telemetry
+2024-12-17 21:16:53,936 DEBUG   SenderThread:75002 [sender.py:send():379] send: tbrecord
+2024-12-17 21:16:53,936 DEBUG   SenderThread:75002 [sender.py:send():379] send: files
+2024-12-17 21:16:53,937 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0 with policy live
+2024-12-17 21:16:54,105 INFO    wandb-upload_2:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/8zd7zwie-events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0
+2024-12-17 21:16:54,402 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/diff.patch
+2024-12-17 21:16:54,403 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:16:54,403 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-metadata.json
+2024-12-17 21:16:54,403 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0
+2024-12-17 21:16:54,403 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/code/FISH/eval_robot.py
+2024-12-17 21:16:54,403 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/requirements.txt
+2024-12-17 21:16:54,403 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/code
+2024-12-17 21:16:54,403 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/code/FISH
+2024-12-17 21:16:54,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:16:55,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:16:56,402 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:16:56,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:16:57,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:16:58,660 DEBUG   SenderThread:75002 [sender.py:send():379] send: config
+2024-12-17 21:16:58,660 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:16:58,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:16:59,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:00,403 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:00,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:01,819 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:02,403 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:02,819 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:03,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:17:03,819 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:04,404 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:04,819 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:05,820 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:06,819 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:07,820 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:08,405 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:08,819 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:17:08,820 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:17:08,820 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:17:08,861 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:09,820 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:10,406 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:10,821 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:11,821 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:12,406 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:12,821 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:13,819 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:17:13,821 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:14,407 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:14,821 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:15,821 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:16,821 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:17,821 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:18,822 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:19,820 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:17:19,821 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:20,822 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:21,822 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:22,822 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:23,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:17:23,818 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:17:23,861 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:24,822 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:24,959 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:17:25,410 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/config.yaml
+2024-12-17 21:17:25,822 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:26,822 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:27,822 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:28,822 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:29,823 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:30,820 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:17:30,823 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:31,823 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:32,823 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:33,823 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:34,823 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:35,823 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:36,820 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:17:36,823 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:37,823 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:38,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:17:38,818 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:17:38,861 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:39,824 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:40,802 DEBUG   SenderThread:75002 [sender.py:send():379] send: files
+2024-12-17 21:17:40,802 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_607b63808c40640027f8.mp4 with policy now
+2024-12-17 21:17:40,803 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:17:40,803 DEBUG   SenderThread:75002 [sender.py:send():379] send: history
+2024-12-17 21:17:40,804 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:17:40,805 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:17:40,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:41,058 INFO    wandb-upload_1:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/htlcjn40-media/videos/eval/0_eval_0_607b63808c40640027f8.mp4
+2024-12-17 21:17:41,414 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json
+2024-12-17 21:17:41,414 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/0_eval_0_607b63808c40640027f8.mp4
+2024-12-17 21:17:41,414 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval
+2024-12-17 21:17:41,414 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos
+2024-12-17 21:17:41,414 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media
+2024-12-17 21:17:41,664 INFO    wandb-upload_2:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/7rdm6uyr-media/videos/eval/0_eval_0_607b63808c40640027f8.mp4
+2024-12-17 21:17:41,824 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:42,414 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:42,810 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:17:42,824 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:43,824 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:44,824 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:45,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:46,415 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:46,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:47,824 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:48,165 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:17:48,416 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:48,824 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:49,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:50,417 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0
+2024-12-17 21:17:50,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:51,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:52,418 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:52,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:53,232 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:17:53,494 DEBUG   SystemMonitor:75002 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-17 21:17:53,495 DEBUG   SenderThread:75002 [sender.py:send():379] send: stats
+2024-12-17 21:17:53,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:17:53,819 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:17:53,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:54,418 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:17:54,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:55,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:56,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:57,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:58,820 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:17:58,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:17:59,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:00,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:01,826 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:02,826 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:03,821 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:18:03,826 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:04,826 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:05,826 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:06,826 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:07,826 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:08,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:18:08,819 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:18:08,861 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:08,909 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:18:09,827 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:10,423 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:18:10,835 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:10,902 DEBUG   SenderThread:75002 [sender.py:send():379] send: files
+2024-12-17 21:18:10,903 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_528c8c53973fc263e4c2.mp4 with policy now
+2024-12-17 21:18:10,906 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:18:10,907 DEBUG   SenderThread:75002 [sender.py:send():379] send: history
+2024-12-17 21:18:10,907 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:18:10,907 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:18:11,136 INFO    wandb-upload_1:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/c3bv692d-media/videos/eval/1_eval_1_528c8c53973fc263e4c2.mp4
+2024-12-17 21:18:11,424 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json
+2024-12-17 21:18:11,424 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/1_eval_1_528c8c53973fc263e4c2.mp4
+2024-12-17 21:18:11,424 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval
+2024-12-17 21:18:11,658 INFO    wandb-upload_2:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/nduniaqo-media/videos/eval/1_eval_1_528c8c53973fc263e4c2.mp4
+2024-12-17 21:18:11,837 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:12,424 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:18:12,839 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:13,424 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:18:13,840 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:14,127 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:18:14,839 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:15,837 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:16,425 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:18:16,839 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:17,837 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:18,426 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:18:18,837 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:19,821 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:18:19,838 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:20,838 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:21,837 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:22,837 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:23,501 DEBUG   SenderThread:75002 [sender.py:send():379] send: stats
+2024-12-17 21:18:23,818 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:18:23,819 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:18:23,865 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:24,838 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:24,907 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:18:25,838 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:26,839 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:27,838 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:28,839 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:29,839 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:30,822 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:18:30,840 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:31,839 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:32,840 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:33,840 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:34,840 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:35,841 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:36,822 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:18:36,845 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:37,843 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:38,819 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:18:38,819 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:18:38,861 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:39,843 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:40,843 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:41,849 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:41,895 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:18:42,843 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:43,844 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:44,844 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:45,248 DEBUG   SenderThread:75002 [sender.py:send():379] send: history
+2024-12-17 21:18:45,248 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:18:45,249 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:18:45,435 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0
+2024-12-17 21:18:45,435 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json
+2024-12-17 21:18:45,660 INFO    wandb-upload_1:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/5dz2mi6h-events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0
+2024-12-17 21:18:45,844 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:46,435 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:18:46,844 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:47,649 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:18:47,844 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:48,436 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:18:48,844 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:49,844 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:50,437 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:18:50,844 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:51,844 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:52,823 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:18:52,845 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:53,496 DEBUG   SenderThread:75002 [sender.py:send():379] send: stats
+2024-12-17 21:18:53,819 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:18:53,819 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:18:53,861 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:54,846 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:55,846 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:56,846 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:57,846 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:57,946 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:18:58,846 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:18:59,846 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:00,440 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:00,863 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:01,859 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:02,189 DEBUG   SenderThread:75002 [sender.py:send():379] send: files
+2024-12-17 21:19:02,189 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_2f040d8fedc4c652052d.mp4 with policy now
+2024-12-17 21:19:02,191 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:19:02,192 DEBUG   SenderThread:75002 [sender.py:send():379] send: history
+2024-12-17 21:19:02,192 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:19:02,192 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:19:02,427 INFO    wandb-upload_2:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/2lqs5gyr-media/videos/eval/2_eval_2_2f040d8fedc4c652052d.mp4
+2024-12-17 21:19:02,440 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:02,441 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json
+2024-12-17 21:19:02,441 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/2_eval_2_2f040d8fedc4c652052d.mp4
+2024-12-17 21:19:02,441 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval
+2024-12-17 21:19:02,689 INFO    wandb-upload_1:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/d359f6i1-media/videos/eval/2_eval_2_2f040d8fedc4c652052d.mp4
+2024-12-17 21:19:02,855 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:03,208 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:19:03,441 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:03,860 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:04,441 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:04,860 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:05,861 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:06,862 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:07,860 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:08,442 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:08,462 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:19:08,819 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:19:08,819 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:19:08,859 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:09,861 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:10,443 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:10,863 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:11,863 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:12,863 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:13,828 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:19:13,866 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:14,863 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:15,866 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:16,867 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:17,880 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:18,829 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:19:18,879 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:19,883 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:20,047 DEBUG   SenderThread:75002 [sender.py:send():379] send: history
+2024-12-17 21:19:20,048 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:19:20,057 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:19:20,446 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0
+2024-12-17 21:19:20,446 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json
+2024-12-17 21:19:20,880 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:21,881 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:22,447 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:22,883 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:23,496 DEBUG   SenderThread:75002 [sender.py:send():379] send: stats
+2024-12-17 21:19:23,826 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:19:23,826 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:19:23,883 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:23,943 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:19:24,448 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:24,886 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:25,886 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:26,448 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:26,887 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:27,888 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:28,449 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:28,886 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:29,833 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:19:29,887 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:30,887 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:31,887 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:32,887 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:33,887 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:34,887 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:35,830 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:19:35,887 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:36,452 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:36,888 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:37,888 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:38,452 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:38,547 DEBUG   SenderThread:75002 [sender.py:send():379] send: files
+2024-12-17 21:19:38,547 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_d96484f372adc32d22ea.mp4 with policy now
+2024-12-17 21:19:38,549 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:19:38,550 DEBUG   SenderThread:75002 [sender.py:send():379] send: history
+2024-12-17 21:19:38,550 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:19:38,550 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:19:38,769 INFO    wandb-upload_2:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/gbinh9n5-media/videos/eval/3_eval_3_d96484f372adc32d22ea.mp4
+2024-12-17 21:19:38,825 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:19:38,825 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:19:38,900 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:39,453 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json
+2024-12-17 21:19:39,453 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/3_eval_3_d96484f372adc32d22ea.mp4
+2024-12-17 21:19:39,453 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval
+2024-12-17 21:19:39,831 INFO    wandb-upload_1:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/3yzpie16-media/videos/eval/3_eval_3_d96484f372adc32d22ea.mp4
+2024-12-17 21:19:39,899 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:40,453 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:40,893 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:40,942 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:19:41,896 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:42,901 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:43,906 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:44,454 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:44,901 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:45,905 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:45,953 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:19:46,455 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:46,905 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:47,917 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:48,924 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:49,905 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:50,913 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:51,839 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:19:51,907 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:52,911 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:53,497 DEBUG   SenderThread:75002 [sender.py:send():379] send: stats
+2024-12-17 21:19:53,828 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:19:53,828 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:19:53,913 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:54,921 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:55,087 DEBUG   SenderThread:75002 [sender.py:send():379] send: history
+2024-12-17 21:19:55,087 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:19:55,088 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:19:55,458 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0
+2024-12-17 21:19:55,458 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json
+2024-12-17 21:19:55,701 INFO    wandb-upload_2:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/4lo3xwe5-events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0
+2024-12-17 21:19:55,922 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:56,458 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:56,922 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:57,081 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:19:57,928 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:58,459 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:19:58,924 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:19:59,930 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:00,459 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:20:00,941 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:01,923 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:02,266 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:20:02,460 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:20:02,923 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:03,923 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:04,923 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:05,923 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:06,923 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:07,833 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:20:07,924 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:08,824 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:20:08,824 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:20:08,924 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:09,924 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:10,924 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:11,924 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:12,463 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:20:12,924 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:13,390 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:20:13,937 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:14,464 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:20:14,619 DEBUG   SenderThread:75002 [sender.py:send():379] send: files
+2024-12-17 21:20:14,619 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_85f8c5555fdc1daec6f7.mp4 with policy now
+2024-12-17 21:20:14,622 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:20:14,622 DEBUG   SenderThread:75002 [sender.py:send():379] send: history
+2024-12-17 21:20:14,622 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:20:14,622 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:20:14,849 INFO    wandb-upload_1:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/1gbzb2e6-media/videos/eval/4_eval_4_85f8c5555fdc1daec6f7.mp4
+2024-12-17 21:20:14,934 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:15,464 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json
+2024-12-17 21:20:15,465 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/4_eval_4_85f8c5555fdc1daec6f7.mp4
+2024-12-17 21:20:15,465 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval
+2024-12-17 21:20:15,729 INFO    wandb-upload_2:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/194n5l0n-media/videos/eval/4_eval_4_85f8c5555fdc1daec6f7.mp4
+2024-12-17 21:20:15,933 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:16,465 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:20:16,937 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:17,935 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:18,836 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:20:18,963 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:19,936 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:20,466 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:20:20,939 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:21,950 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:22,467 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:20:22,942 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:23,497 DEBUG   SenderThread:75002 [sender.py:send():379] send: stats
+2024-12-17 21:20:23,837 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:20:23,837 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:20:23,950 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:23,954 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:20:24,938 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:25,949 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:26,977 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:27,948 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:28,955 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:29,850 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:20:29,961 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:30,952 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:31,955 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:32,953 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:33,956 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:34,957 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:35,121 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:20:35,131 DEBUG   SenderThread:75002 [sender.py:send():379] send: history
+2024-12-17 21:20:35,131 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:20:35,132 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:20:35,165 DEBUG   SenderThread:75002 [sender.py:send():379] send: exit
+2024-12-17 21:20:35,165 INFO    SenderThread:75002 [sender.py:send_exit():586] handling exit code: 0
+2024-12-17 21:20:35,166 INFO    SenderThread:75002 [sender.py:send_exit():588] handling runtime: 221
+2024-12-17 21:20:35,166 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:20:35,166 INFO    SenderThread:75002 [sender.py:send_exit():594] send defer
+2024-12-17 21:20:35,166 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:35,166 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-17 21:20:35,166 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:35,166 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-17 21:20:35,166 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 1
+2024-12-17 21:20:35,166 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:35,166 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-17 21:20:35,167 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:35,167 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-17 21:20:35,167 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 2
+2024-12-17 21:20:35,167 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:35,167 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-17 21:20:35,167 INFO    HandlerThread:75002 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-17 21:20:35,167 DEBUG   SystemMonitor:75002 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-17 21:20:35,167 DEBUG   SystemMonitor:75002 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-17 21:20:35,167 INFO    HandlerThread:75002 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-17 21:20:35,168 INFO    HandlerThread:75002 [interfaces.py:finish():200] Joined disk monitor
+2024-12-17 21:20:35,217 INFO    HandlerThread:75002 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-17 21:20:35,218 INFO    HandlerThread:75002 [interfaces.py:finish():200] Joined memory monitor
+2024-12-17 21:20:35,218 INFO    HandlerThread:75002 [interfaces.py:finish():200] Joined network monitor
+2024-12-17 21:20:35,218 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:35,218 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-17 21:20:35,218 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 3
+2024-12-17 21:20:35,218 DEBUG   SenderThread:75002 [sender.py:send():379] send: stats
+2024-12-17 21:20:35,219 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:35,219 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-17 21:20:35,219 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:35,219 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-17 21:20:35,219 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 4
+2024-12-17 21:20:35,219 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:35,219 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-17 21:20:35,471 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0
+2024-12-17 21:20:35,471 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json
+2024-12-17 21:20:36,471 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:20:41,220 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:20:41,220 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:20:41,220 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:41,221 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-17 21:20:41,221 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-17 21:20:41,221 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 5
+2024-12-17 21:20:41,221 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:20:41,221 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:41,221 DEBUG   SenderThread:75002 [sender.py:send():379] send: history
+2024-12-17 21:20:41,221 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-17 21:20:41,222 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:20:41,222 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:20:41,222 DEBUG   SenderThread:75002 [sender.py:send():379] send: summary
+2024-12-17 21:20:41,226 INFO    SenderThread:75002 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:20:41,226 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:41,226 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-17 21:20:41,226 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 6
+2024-12-17 21:20:41,226 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:41,226 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-17 21:20:41,226 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:41,227 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-17 21:20:41,227 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 7
+2024-12-17 21:20:41,227 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:20:41,227 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:41,227 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-17 21:20:41,227 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:41,227 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-17 21:20:41,473 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json
+2024-12-17 21:20:42,166 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:20:42,831 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 8
+2024-12-17 21:20:42,831 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:20:42,831 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:42,832 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-17 21:20:42,832 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:42,832 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-17 21:20:42,832 INFO    SenderThread:75002 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-17 21:20:42,834 INFO    SenderThread:75002 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-17 21:20:42,849 INFO    SenderThread:75002 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-17 21:20:42,859 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 9
+2024-12-17 21:20:42,859 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:42,859 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-17 21:20:42,860 DEBUG   SenderThread:75002 [sender.py:send():379] send: artifact
+2024-12-17 21:20:43,167 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:20:43,275 INFO    wandb-upload_0:75002 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp58wukwcy
+2024-12-17 21:20:43,276 INFO    wandb-upload_2:75002 [upload_job.py:push():85] Skipped uploading /tmp/tmpyn9jqz0t/wandb-job.json
+2024-12-17 21:20:43,474 INFO    Thread-12 :75002 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:20:43,479 INFO    wandb-upload_1:75002 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpvalkza0y
+2024-12-17 21:20:44,190 INFO    SenderThread:75002 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM4MzE5ODcyNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM4MzE0NTc2Mg==', 'versionIndex': 377}}}
+2024-12-17 21:20:44,190 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:44,190 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-17 21:20:44,190 INFO    SenderThread:75002 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-17 21:20:44,475 INFO    SenderThread:75002 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files
+2024-12-17 21:20:44,475 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/config.yaml config.yaml
+2024-12-17 21:20:44,475 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0 events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0
+2024-12-17 21:20:44,477 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json wandb-summary.json
+2024-12-17 21:20:44,477 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/requirements.txt requirements.txt
+2024-12-17 21:20:44,477 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-metadata.json wandb-metadata.json
+2024-12-17 21:20:44,477 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log output.log
+2024-12-17 21:20:44,477 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/diff.patch diff.patch
+2024-12-17 21:20:44,477 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-17 21:20:44,483 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/3_eval_3_d96484f372adc32d22ea.mp4 media/videos/eval/3_eval_3_d96484f372adc32d22ea.mp4
+2024-12-17 21:20:44,483 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/0_eval_0_607b63808c40640027f8.mp4 media/videos/eval/0_eval_0_607b63808c40640027f8.mp4
+2024-12-17 21:20:44,484 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/1_eval_1_528c8c53973fc263e4c2.mp4 media/videos/eval/1_eval_1_528c8c53973fc263e4c2.mp4
+2024-12-17 21:20:44,484 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/2_eval_2_2f040d8fedc4c652052d.mp4 media/videos/eval/2_eval_2_2f040d8fedc4c652052d.mp4
+2024-12-17 21:20:44,484 INFO    SenderThread:75002 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/media/videos/eval/4_eval_4_85f8c5555fdc1daec6f7.mp4 media/videos/eval/4_eval_4_85f8c5555fdc1daec6f7.mp4
+2024-12-17 21:20:44,484 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 10
+2024-12-17 21:20:44,484 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:20:44,484 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:44,484 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-17 21:20:44,484 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:44,484 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-17 21:20:44,484 INFO    SenderThread:75002 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 21:20:44,647 INFO    wandb-upload_3:75002 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/wandb-summary.json
+2024-12-17 21:20:44,653 INFO    wandb-upload_0:75002 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/config.yaml
+2024-12-17 21:20:44,691 INFO    wandb-upload_4:75002 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/output.log
+2024-12-17 21:20:44,695 INFO    wandb-upload_2:75002 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/files/requirements.txt
+2024-12-17 21:20:44,698 INFO    wandb-upload_1:75002 [upload_job.py:push():130] Uploaded file /tmp/tmp_bvp7plcwandb/eajpd6gl-events.out.tfevents.1734488213.leonmkim-ROG-Strix-G15CS-G15CS.74816.0
+2024-12-17 21:20:44,898 INFO    Thread-11 (_thread_body):75002 [sender.py:transition_state():614] send defer: 11
+2024-12-17 21:20:44,899 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:44,899 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-17 21:20:44,899 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:44,899 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-17 21:20:44,899 INFO    SenderThread:75002 [file_pusher.py:join():175] waiting for file pusher
+2024-12-17 21:20:44,900 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 12
+2024-12-17 21:20:44,900 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:44,900 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-17 21:20:44,900 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:44,900 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-17 21:20:44,900 INFO    SenderThread:75002 [file_stream.py:finish():601] file stream finish called
+2024-12-17 21:20:44,962 INFO    SenderThread:75002 [file_stream.py:finish():605] file stream finish is done
+2024-12-17 21:20:44,962 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 13
+2024-12-17 21:20:44,962 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:44,962 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-17 21:20:44,962 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:44,962 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-17 21:20:44,962 INFO    SenderThread:75002 [sender.py:transition_state():614] send defer: 14
+2024-12-17 21:20:44,962 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:20:44,963 DEBUG   SenderThread:75002 [sender.py:send():379] send: final
+2024-12-17 21:20:44,963 INFO    HandlerThread:75002 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-17 21:20:44,963 DEBUG   SenderThread:75002 [sender.py:send():379] send: footer
+2024-12-17 21:20:44,963 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:20:44,963 INFO    SenderThread:75002 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-17 21:20:44,963 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:20:44,963 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:20:44,964 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: server_info
+2024-12-17 21:20:44,964 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: server_info
+2024-12-17 21:20:44,965 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:20:44,965 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-17 21:20:44,966 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-17 21:20:44,966 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:20:45,040 DEBUG   SenderThread:75002 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:20:45,040 INFO    MainThread:75002 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-17 21:20:45,041 INFO    MainThread:75002 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-17 21:20:45,041 INFO    MainThread:75002 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-17 21:20:45,041 DEBUG   HandlerThread:75002 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-17 21:20:45,041 INFO    HandlerThread:75002 [handler.py:finish():882] shutting down handler
+2024-12-17 21:20:45,966 INFO    WriterThread:75002 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/run-14r6tvph.wandb
+2024-12-17 21:20:46,040 INFO    SenderThread:75002 [sender.py:finish():1615] shutting down sender
+2024-12-17 21:20:46,041 INFO    SenderThread:75002 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 21:20:46,041 INFO    SenderThread:75002 [file_pusher.py:join():175] waiting for file pusher

211647/wandb/run-20241217_211653-14r6tvph/logs/debug.log

@@ -0,0 +1,31 @@
+2024-12-17 21:16:53,162 INFO    MainThread:74816 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-17 21:16:53,162 INFO    MainThread:74816 [wandb_setup.py:_flush():76] Configure stats pid to 74816
+2024-12-17 21:16:53,162 INFO    MainThread:74816 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-17 21:16:53,162 INFO    MainThread:74816 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/settings
+2024-12-17 21:16:53,162 INFO    MainThread:74816 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-17 21:16:53,162 INFO    MainThread:74816 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-17 21:16:53,162 INFO    MainThread:74816 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-17 21:16:53,162 INFO    MainThread:74816 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-17 21:16:53,162 INFO    MainThread:74816 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/logs/debug.log
+2024-12-17 21:16:53,162 INFO    MainThread:74816 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/wandb/run-20241217_211653-14r6tvph/logs/debug-internal.log
+2024-12-17 21:16:53,163 INFO    MainThread:74816 [wandb_init.py:init():569] calling init triggers
+2024-12-17 21:16:53,164 INFO    MainThread:74816 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '508_0', 'wandb_notes': '508_0_req_579_0', 'eval': True, 'true_action_history': False, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': False, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['fowlers_twodim'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/508_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647'}
+2024-12-17 21:16:53,164 INFO    MainThread:74816 [wandb_init.py:init():619] starting backend
+2024-12-17 21:16:53,164 INFO    MainThread:74816 [wandb_init.py:init():623] setting up manager
+2024-12-17 21:16:53,170 INFO    MainThread:74816 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-17 21:16:53,172 INFO    MainThread:74816 [wandb_init.py:init():631] backend started and connected
+2024-12-17 21:16:53,182 INFO    MainThread:74816 [wandb_init.py:init():720] updated telemetry
+2024-12-17 21:16:53,189 INFO    MainThread:74816 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-17 21:16:53,413 INFO    MainThread:74816 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-17 21:16:53,482 INFO    MainThread:74816 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-17 21:16:53,482 INFO    MainThread:74816 [wandb_init.py:init():804] starting run threads in backend
+2024-12-17 21:16:53,817 INFO    MainThread:74816 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-17 21:16:53,818 INFO    MainThread:74816 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-17 21:16:53,818 INFO    MainThread:74816 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-17 21:16:53,818 INFO    MainThread:74816 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-17 21:16:53,819 INFO    MainThread:74816 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-17 21:16:53,819 INFO    MainThread:74816 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/508_0/211647/tb, True
+2024-12-17 21:16:58,659 INFO    MainThread:74816 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'}
+2024-12-17 21:20:46,044 WARNING MsgRouterThr:74816 [router.py:message_loop():77] message_loop has been closed

config.yaml

@@ -0,0 +1,546 @@
+root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data
+replay_buffer_size: 150000
+replay_buffer_num_workers: 2
+nstep: 3
+batch_size: 128
+seed: 0
+dataset_shuffle_seed: 0
+device: cuda
+save_video: true
+save_train_video: true
+use_tb: true
+use_wandb: true
+wandb_run_id: '508_0'
+wandb_notes: 508_0_req_579_0
+eval: false
+true_action_history: false
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.2
+env_dtc_max_value: 0.4
+grasped_normals_mask_max_dtc_value: 0.2
+env_normals_mask_max_dtc_value: 0.4
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: true
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type: small
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+baseline: false
+train_demo_idxs_list_or_num: -1
+log_train_every_steps: 25
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+store_dataset_in_memory: false
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+include_groups_list:
+- fowlers_twodim
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+name_of_valid_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr
+valid_demo_idxs_list_or_num: null
+val_num_groups: 0
+load_bc: ${agent.load_checkpoint}
+checkpoint_epoch_list:
+- 99
+- 199
+- 299
+- 399
+- 499
+- 599
+- 699
+- 799
+- 899
+- 999
+- 1249
+- 1499
+- 1749
+- 1999
+- 2999
+- 3999
+- 4999
+- 5999
+- 6999
+- 7999
+- 8999
+- 9999
+snapshot_root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data/FISH
+save_snapshot: true
+save_last_snapshot: true
+save_snapshot_when_done: true
+top_k_checkpoints: 5
+save_snapshot_link_to_weights_dir: deprecated
+bc_regularize: false
+bc_weight_type: qfilter
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: 6
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+      input_shapes:
+        observation.image:
+        - 13
+        - 180
+        - 240
+        context_observation.image:
+        - 13
+        - 180
+        - 240
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: cosine
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: 0.0001
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 1000000
+  num_seed_frames: 0
+  num_train_epochs: 15000
+  validate_every_epochs: 250
+  validate_diffusion_on_action_loss_every_epochs: 250
+  train_eval_diffusion_on_action_loss_every_epochs: 250
+  check_topk_every_epochs: 250
+  save_snapshot_every_epochs: 1500
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: ${device}
+      cam_resize_shape: ${agent.desired_image_shape}
+      orig_cam_shape: ${agent.orig_cam_shape}
+      policy_frequency: ${policy_frequency}
+      interpolation_frequency: ${interpolation_frequency}
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: ${agent.config.policy_cfg.horizon}
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: ${agent.config.policy_cfg.kernel_size}
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: ${num_train_frames_diffusion}
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: ${use_depth}
+        use_color: ${use_color}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_masks}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${color_crop_type}
+          depth_crop_type: ${depth_crop_type}
+          segmask_crop_type: ${segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${add_crop_binary_mask}
+          add_coord_conv_map: ${add_coord_conv_map}
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: ${use_context_color}
+          use_depth: ${use_context_depth}
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: ${use_context_segmask}
+            representation: ${mask_representation}
+            mask_list: ${mask_list}
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: ${context_color_crop_type}
+            depth_crop_type: ${context_depth_crop_type}
+            segmask_crop_type: ${context_segmask_crop_type}
+            crop_hw: ${crop_hw}
+            crop_down_offset: ${crop_down_offset}
+            add_crop_binary_mask: ${context_add_crop_binary_mask}
+            add_coord_conv_map: ${context_add_coord_conv_map}
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: ${use_contact_map}
+        use_sdf_maps: ${use_sdf_maps}
+        use_normals_maps: ${use_normals_maps}
+        which_objects: ${which_objects}
+        grasped_dtc_max_value: ${grasped_dtc_max_value}
+        env_dtc_max_value: ${env_dtc_max_value}
+        grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+        env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+        clamp_dtc: ${clamp_dtc}
+        max_contact_prob: ${max_contact_prob}
+        mask_normals_within_sdf: ${mask_normals_within_sdf}
+        dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+        adaptive_normals_mask: ${adaptive_normals_mask}
+        max_depth: ${max_depth}
+        image_shape: ${agent.desired_image_shape}
+        learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: ${contact_model_name}
+        zero_centered: false
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 15000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 250
+validate_diffusion_on_action_loss_every_epochs: 250
+train_eval_diffusion_on_action_loss_every_epochs: 250
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: 250
+save_snapshot_every_epochs_diffusion: 1500
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1