Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,18 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/0_eval_0_479c2ab590e51b63c1e7.mp4 filter=lfs diff=lfs merge=lfs -text
+205031/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/4_eval_4_cb4b291fcf155dd3f366.mp4 filter=lfs diff=lfs merge=lfs -text
+205031/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+205031/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+205031/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+205031/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/3_eval_3_c5032837d4220bef66c0.mp4 filter=lfs diff=lfs merge=lfs -text
+205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/1_eval_1_3dd750e4041b7d940696.mp4 filter=lfs diff=lfs merge=lfs -text
+205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/2_eval_2_41e781c681092c8d0dae.mp4 filter=lfs diff=lfs merge=lfs -text
+205031/episode_rosbags/episode_2_2024-12-17-20-52-27.bag filter=lfs diff=lfs merge=lfs -text
+205031/episode_rosbags/episode_4_2024-12-17-20-53-47.bag filter=lfs diff=lfs merge=lfs -text
+205031/episode_rosbags/episode_1_2024-12-17-20-51-47.bag filter=lfs diff=lfs merge=lfs -text
+205031/episode_rosbags/episode_3_2024-12-17-20-53-07.bag filter=lfs diff=lfs merge=lfs -text
+205031/episode_rosbags/episode_0_2024-12-17-20-51-07.bag filter=lfs diff=lfs merge=lfs -text

205030/.hydra/config.yaml

@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '505_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${now:%H%M%S}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

205030/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_policy
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205030
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

205030/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion

205030/eval_policy.log

@@ -0,0 +1,15 @@
+[2024-12-17 20:50:30,157][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-17 20:50:30,160][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2024-12-17 20:50:33,110][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+

205031/.hydra/config.yaml

@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '505_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${now:%H%M%S}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

205031/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

205031/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion

205031/episode_rosbags/aligned_depth_to_color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

205031/episode_rosbags/cam_tf_world.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256

205031/episode_rosbags/color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

205031/episode_rosbags/depth_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200

205031/episode_rosbags/episode_0_2024-12-17-20-51-07.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e13d7d46a0575e9dfd1110a5543544120b3059c7400c5f6d68150087f9cf9e1
+size 1330712825

205031/episode_rosbags/episode_1_2024-12-17-20-51-47.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:612effd61c51cd34b30e97caf414cdf6c6d7be8c0533e0976acafcab68e1c70d
+size 1335894396

205031/episode_rosbags/episode_2_2024-12-17-20-52-27.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e7e6ffa23cfb70f6ae5a33e79e54572781153e582adc491067249ca6e8ab100
+size 1330234920

205031/episode_rosbags/episode_3_2024-12-17-20-53-07.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f49bf4227dd698785788970c3d780836042599129b7d446dc2a2d806666c7123
+size 1332718732

205031/episode_rosbags/episode_4_2024-12-17-20-53-47.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6e44c1283b8383c490d774f8b19b12b8b8b81f34eccb7e1984af53309b9d498
+size 1332635483

205031/eval_robot.log

@@ -0,0 +1,12 @@
+[2024-12-17 20:50:31,675][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-17 20:50:31,678][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+

205031/eval_video/0_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:479c2ab590e51b63c1e7d102b999d930586a97b94c8e0c6eb58a8bb8570849d6
+size 1117369

205031/eval_video/1_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3dd750e4041b7d94069680d616adf51ded3f6bd3bdb5933c3aac378a2db1dcbe
+size 1120651

205031/eval_video/2_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41e781c681092c8d0daee8143fbd4dd2873d760169b212b4a595d5eec4a4fa6f
+size 1118222

205031/eval_video/3_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5032837d4220bef66c08b91bf3175322db59696f748d5d0a44380be57a27b65
+size 1117547

205031/eval_video/4_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb4b291fcf155dd3f3662521ca561d4594281a40887c90201c519e5954662247
+size 1182110

205031/tb/events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49261be59278a3e4b99bc676906629f885d4b48634788cf19a9ac21b4fbf7195
+size 1103

205031/wandb/debug-internal.log

@@ -0,0 +1,659 @@
+2024-12-17 20:50:37,561 INFO    StreamThr :69178 [internal.py:wandb_internal():85] W&B internal server running at pid: 69178, started at: 2024-12-17 20:50:37.560794
+2024-12-17 20:50:37,561 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status
+2024-12-17 20:50:37,573 INFO    WriterThread:69178 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/run-v5toezqj.wandb
+2024-12-17 20:50:37,573 DEBUG   SenderThread:69178 [sender.py:send():379] send: header
+2024-12-17 20:50:37,580 DEBUG   SenderThread:69178 [sender.py:send():379] send: run
+2024-12-17 20:50:37,815 INFO    SenderThread:69178 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files
+2024-12-17 20:50:37,815 INFO    SenderThread:69178 [sender.py:_start_run_threads():1188] run started: v5toezqj with start time 1734486637.561221
+2024-12-17 20:50:37,831 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: check_version
+2024-12-17 20:50:37,831 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: check_version
+2024-12-17 20:50:37,909 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: run_start
+2024-12-17 20:50:37,914 DEBUG   HandlerThread:69178 [system_info.py:__init__():26] System info init
+2024-12-17 20:50:37,914 DEBUG   HandlerThread:69178 [system_info.py:__init__():41] System info init done
+2024-12-17 20:50:37,914 INFO    HandlerThread:69178 [system_monitor.py:start():194] Starting system monitor
+2024-12-17 20:50:37,914 INFO    SystemMonitor:69178 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-17 20:50:37,914 INFO    HandlerThread:69178 [system_monitor.py:probe():214] Collecting system info
+2024-12-17 20:50:37,915 INFO    SystemMonitor:69178 [interfaces.py:start():188] Started cpu monitoring
+2024-12-17 20:50:37,915 INFO    SystemMonitor:69178 [interfaces.py:start():188] Started disk monitoring
+2024-12-17 20:50:37,916 INFO    SystemMonitor:69178 [interfaces.py:start():188] Started gpu monitoring
+2024-12-17 20:50:37,916 INFO    SystemMonitor:69178 [interfaces.py:start():188] Started memory monitoring
+2024-12-17 20:50:37,917 INFO    SystemMonitor:69178 [interfaces.py:start():188] Started network monitoring
+2024-12-17 20:50:37,929 DEBUG   HandlerThread:69178 [system_info.py:probe():152] Probing system
+2024-12-17 20:50:37,930 DEBUG   HandlerThread:69178 [system_info.py:_probe_git():137] Probing git
+2024-12-17 20:50:37,935 DEBUG   HandlerThread:69178 [system_info.py:_probe_git():145] Probing git done
+2024-12-17 20:50:37,935 DEBUG   HandlerThread:69178 [system_info.py:probe():200] Probing system done
+2024-12-17 20:50:37,935 DEBUG   HandlerThread:69178 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-18T01:50:37.929248', 'startedAt': '2024-12-18T01:50:37.553164', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '5f28b54373d7d41e4ebace380ff1080a8ffc4051'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.51775, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.895, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.968, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 360.83281326293945}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2024-12-17 20:50:37,935 INFO    HandlerThread:69178 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-17 20:50:37,935 INFO    HandlerThread:69178 [system_monitor.py:probe():227] Publishing system info
+2024-12-17 20:50:37,936 DEBUG   HandlerThread:69178 [system_info.py:_save_code():44] Saving code
+2024-12-17 20:50:37,937 DEBUG   HandlerThread:69178 [system_info.py:_save_code():67] Saving code done
+2024-12-17 20:50:37,937 DEBUG   HandlerThread:69178 [system_info.py:_save_patches():84] Saving git patches
+2024-12-17 20:50:38,004 DEBUG   HandlerThread:69178 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-17 20:50:38,005 INFO    HandlerThread:69178 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-17 20:50:38,008 DEBUG   SenderThread:69178 [sender.py:send():379] send: files
+2024-12-17 20:50:38,008 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-17 20:50:38,008 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-17 20:50:38,008 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-17 20:50:38,246 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-17 20:50:38,246 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: python_packages
+2024-12-17 20:50:38,246 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:50:38,247 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:50:38,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:38,293 INFO    HandlerThread:69178 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/tb"
+  save: true
+}
+control {
+  relay_id: "f5f4f6fa4a0e46c6a826af22cb762514"
+}
+_info {
+  stream_id: "v5toezqj"
+}
+
+2024-12-17 20:50:38,293 DEBUG   HandlerThread:69178 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-17 20:50:38,367 DEBUG   SenderThread:69178 [sender.py:send():379] send: telemetry
+2024-12-17 20:50:38,368 DEBUG   SenderThread:69178 [sender.py:send():379] send: tbrecord
+2024-12-17 20:50:38,368 DEBUG   SenderThread:69178 [sender.py:send():379] send: files
+2024-12-17 20:50:38,368 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0 with policy live
+2024-12-17 20:50:38,386 INFO    wandb-upload_2:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/1louuc8s-diff.patch
+2024-12-17 20:50:38,387 INFO    wandb-upload_0:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/mtmc0peb-wandb-metadata.json
+2024-12-17 20:50:38,387 INFO    wandb-upload_1:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/i4p6vweu-code/FISH/eval_robot.py
+2024-12-17 20:50:38,570 INFO    wandb-upload_3:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/fq8htgz7-events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0
+2024-12-17 20:50:38,817 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/code/FISH/eval_robot.py
+2024-12-17 20:50:38,817 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-metadata.json
+2024-12-17 20:50:38,817 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/requirements.txt
+2024-12-17 20:50:38,818 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/diff.patch
+2024-12-17 20:50:38,818 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:50:38,818 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0
+2024-12-17 20:50:38,818 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/code
+2024-12-17 20:50:38,818 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/code/FISH
+2024-12-17 20:50:39,247 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:40,247 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:40,817 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:50:41,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:42,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:43,247 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:50:43,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:44,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:45,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:46,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:47,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:48,247 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:50:48,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:49,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:50,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:51,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:51,545 DEBUG   SenderThread:69178 [sender.py:send():379] send: config
+2024-12-17 20:50:52,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:52,820 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:50:53,246 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:50:53,247 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:50:53,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:53,369 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:50:54,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:54,820 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:50:55,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:56,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:56,821 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:50:57,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:58,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:58,822 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:50:58,978 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:50:59,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:00,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:00,822 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:51:01,250 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:02,250 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:03,251 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:04,153 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:51:04,250 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:05,250 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:06,251 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:06,824 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:51:07,251 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:08,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:51:08,249 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:51:08,293 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:08,824 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:51:09,251 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:09,400 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:51:09,825 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/config.yaml
+2024-12-17 20:51:10,252 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:10,825 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:51:11,252 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:12,253 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:13,253 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:14,253 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:15,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:51:15,253 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:16,253 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:17,253 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:18,253 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:19,253 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:20,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:51:20,254 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:21,254 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:22,254 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:23,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:51:23,249 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:51:23,293 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:24,254 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:25,254 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:25,350 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:51:26,254 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:27,254 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:28,255 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:29,255 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:30,255 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:31,250 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:51:31,255 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:32,255 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:33,255 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:34,255 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:35,255 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:35,589 DEBUG   SenderThread:69178 [sender.py:send():379] send: files
+2024-12-17 20:51:35,589 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_479c2ab590e51b63c1e7.mp4 with policy now
+2024-12-17 20:51:35,589 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 20:51:35,590 DEBUG   SenderThread:69178 [sender.py:send():379] send: history
+2024-12-17 20:51:35,590 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:51:35,591 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:51:35,832 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/0_eval_0_479c2ab590e51b63c1e7.mp4
+2024-12-17 20:51:35,832 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json
+2024-12-17 20:51:35,832 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media
+2024-12-17 20:51:35,832 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos
+2024-12-17 20:51:35,832 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval
+2024-12-17 20:51:35,858 INFO    wandb-upload_0:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/aemlia3l-media/videos/eval/0_eval_0_479c2ab590e51b63c1e7.mp4
+2024-12-17 20:51:36,164 INFO    wandb-upload_1:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/370593m2-media/videos/eval/0_eval_0_479c2ab590e51b63c1e7.mp4
+2024-12-17 20:51:36,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:36,592 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:51:36,832 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:51:37,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:37,917 DEBUG   SystemMonitor:69178 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-17 20:51:37,917 DEBUG   SenderThread:69178 [sender.py:send():379] send: stats
+2024-12-17 20:51:38,252 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:51:38,252 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:51:38,293 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:39,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:40,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:40,833 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:51:41,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:41,804 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:51:42,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:42,834 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:51:43,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:43,834 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0
+2024-12-17 20:51:44,122 INFO    wandb-upload_2:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/2lsrgr80-events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0
+2024-12-17 20:51:44,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:44,835 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:51:45,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:46,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:46,835 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:51:47,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:47,398 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:51:48,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:48,836 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:51:49,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:50,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:51,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:52,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:53,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:51:53,250 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:51:53,250 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:51:53,293 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:54,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:55,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:56,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:57,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:58,250 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:51:58,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:59,258 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:00,258 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:01,258 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:02,258 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:03,250 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:52:03,258 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:04,258 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:05,258 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:06,258 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:07,259 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:07,918 DEBUG   SenderThread:69178 [sender.py:send():379] send: stats
+2024-12-17 20:52:08,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:52:08,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:52:08,257 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:52:08,301 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:09,259 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:10,259 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:11,259 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:12,259 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:13,260 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:14,130 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:52:14,260 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:15,262 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:15,570 DEBUG   SenderThread:69178 [sender.py:send():379] send: files
+2024-12-17 20:52:15,570 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_3dd750e4041b7d940696.mp4 with policy now
+2024-12-17 20:52:15,576 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 20:52:15,577 DEBUG   SenderThread:69178 [sender.py:send():379] send: history
+2024-12-17 20:52:15,577 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:52:15,578 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:52:15,832 INFO    wandb-upload_3:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/q6lbjbni-media/videos/eval/1_eval_1_3dd750e4041b7d940696.mp4
+2024-12-17 20:52:15,844 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json
+2024-12-17 20:52:15,845 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/1_eval_1_3dd750e4041b7d940696.mp4
+2024-12-17 20:52:15,845 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval
+2024-12-17 20:52:16,129 INFO    wandb-upload_0:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/inolq7gm-media/videos/eval/1_eval_1_3dd750e4041b7d940696.mp4
+2024-12-17 20:52:16,263 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:16,845 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:52:17,262 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:17,845 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:52:18,263 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:19,137 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:52:19,267 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:20,263 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:20,846 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:52:21,270 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:22,263 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:22,847 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:52:23,251 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:52:23,251 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:52:23,293 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:23,681 DEBUG   SenderThread:69178 [sender.py:send():379] send: history
+2024-12-17 20:52:23,681 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:52:23,703 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:52:23,847 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0
+2024-12-17 20:52:23,847 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json
+2024-12-17 20:52:24,264 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:24,294 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:52:24,848 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:52:25,269 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:26,267 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:26,848 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:52:27,266 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:28,266 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:28,849 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:52:29,266 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:30,254 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:52:30,266 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:31,266 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:32,267 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:33,267 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:34,267 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:35,267 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:36,254 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:52:36,267 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:37,267 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:37,919 DEBUG   SenderThread:69178 [sender.py:send():379] send: stats
+2024-12-17 20:52:38,251 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:52:38,251 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:52:38,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:39,267 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:40,268 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:41,268 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:41,337 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:52:42,268 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:43,268 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:44,268 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:45,268 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:46,268 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:47,255 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:52:47,268 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:48,268 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:49,268 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:50,269 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:51,269 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:52,269 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:53,251 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:52:53,251 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:52:53,293 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:53,398 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:52:54,274 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:54,857 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:52:55,182 DEBUG   SenderThread:69178 [sender.py:send():379] send: files
+2024-12-17 20:52:55,182 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_41e781c681092c8d0dae.mp4 with policy now
+2024-12-17 20:52:55,182 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 20:52:55,183 DEBUG   SenderThread:69178 [sender.py:send():379] send: history
+2024-12-17 20:52:55,183 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:52:55,183 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:52:55,274 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:55,446 INFO    wandb-upload_1:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/rklbn3xb-media/videos/eval/2_eval_2_41e781c681092c8d0dae.mp4
+2024-12-17 20:52:55,858 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json
+2024-12-17 20:52:55,859 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/2_eval_2_41e781c681092c8d0dae.mp4
+2024-12-17 20:52:55,859 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval
+2024-12-17 20:52:56,129 INFO    wandb-upload_2:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/hbutwpli-media/videos/eval/2_eval_2_41e781c681092c8d0dae.mp4
+2024-12-17 20:52:56,281 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:56,859 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:52:57,289 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:57,859 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:52:58,276 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:58,711 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:52:59,281 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:00,295 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:00,860 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:01,292 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:02,286 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:02,861 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:03,291 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:03,861 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0
+2024-12-17 20:53:04,105 INFO    wandb-upload_3:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/dvaq4j8i-events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0
+2024-12-17 20:53:04,298 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:04,510 DEBUG   SenderThread:69178 [sender.py:send():379] send: history
+2024-12-17 20:53:04,510 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:53:04,511 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:53:04,511 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:53:04,862 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json
+2024-12-17 20:53:04,862 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:05,293 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:06,295 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:06,862 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:07,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:07,919 DEBUG   SenderThread:69178 [sender.py:send():379] send: stats
+2024-12-17 20:53:08,252 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:53:08,252 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:53:08,294 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:08,863 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:09,296 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:09,937 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:53:10,296 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:10,864 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:11,296 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:12,296 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:13,296 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:14,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:14,865 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:15,255 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:53:15,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:16,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:17,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:18,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:19,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:20,255 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:53:20,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:21,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:22,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:23,252 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:53:23,252 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:53:23,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:24,298 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:25,298 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:25,414 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:53:26,298 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:27,298 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:28,298 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:29,298 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:30,298 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:31,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:53:31,298 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:32,298 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:33,299 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:34,299 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:35,301 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:35,372 DEBUG   SenderThread:69178 [sender.py:send():379] send: files
+2024-12-17 20:53:35,372 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_c5032837d4220bef66c0.mp4 with policy now
+2024-12-17 20:53:35,413 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 20:53:35,414 DEBUG   SenderThread:69178 [sender.py:send():379] send: history
+2024-12-17 20:53:35,414 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:53:35,415 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:53:35,629 INFO    wandb-upload_0:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/us394crz-media/videos/eval/3_eval_3_c5032837d4220bef66c0.mp4
+2024-12-17 20:53:35,872 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json
+2024-12-17 20:53:35,872 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/3_eval_3_c5032837d4220bef66c0.mp4
+2024-12-17 20:53:35,872 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval
+2024-12-17 20:53:36,136 INFO    wandb-upload_1:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/c8thr94g-media/videos/eval/3_eval_3_c5032837d4220bef66c0.mp4
+2024-12-17 20:53:36,309 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:36,415 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:53:36,872 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:37,306 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:37,919 DEBUG   SenderThread:69178 [sender.py:send():379] send: stats
+2024-12-17 20:53:38,269 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:53:38,269 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:53:38,352 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:39,316 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:40,318 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:40,874 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:41,330 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:41,718 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:53:42,331 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:42,874 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:43,328 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:43,875 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0
+2024-12-17 20:53:44,327 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:44,557 DEBUG   SenderThread:69178 [sender.py:send():379] send: history
+2024-12-17 20:53:44,557 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:53:44,558 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:53:44,875 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json
+2024-12-17 20:53:44,875 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:45,319 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:46,319 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:46,876 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:47,327 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:47,452 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:53:48,320 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:48,876 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:49,341 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:50,321 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:50,877 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:51,321 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:52,321 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:53,252 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:53:53,252 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:53:53,322 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:53,406 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:53:54,321 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:55,322 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:56,322 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:57,322 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:58,322 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:58,880 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:59,258 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:53:59,322 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:00,322 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:01,322 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:02,322 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:03,323 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:04,323 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:05,258 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:54:05,323 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:06,323 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:07,323 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:07,920 DEBUG   SenderThread:69178 [sender.py:send():379] send: stats
+2024-12-17 20:54:08,252 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:54:08,252 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:54:08,324 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:08,883 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:54:09,324 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:10,324 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:10,391 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:54:11,324 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:12,324 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:13,324 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:14,324 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:15,324 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:15,940 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:54:16,325 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:16,886 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:54:17,032 DEBUG   SenderThread:69178 [sender.py:send():379] send: files
+2024-12-17 20:54:17,032 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_cb4b291fcf155dd3f366.mp4 with policy now
+2024-12-17 20:54:17,035 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 20:54:17,035 DEBUG   SenderThread:69178 [sender.py:send():379] send: history
+2024-12-17 20:54:17,035 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:54:17,036 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:54:17,297 INFO    wandb-upload_2:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/07e8vmtr-media/videos/eval/4_eval_4_cb4b291fcf155dd3f366.mp4
+2024-12-17 20:54:17,331 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:17,886 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json
+2024-12-17 20:54:17,886 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/4_eval_4_cb4b291fcf155dd3f366.mp4
+2024-12-17 20:54:17,887 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval
+2024-12-17 20:54:18,176 INFO    wandb-upload_3:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/c23rhxtl-media/videos/eval/4_eval_4_cb4b291fcf155dd3f366.mp4
+2024-12-17 20:54:18,336 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:18,887 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:54:19,335 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:20,330 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:21,335 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:21,617 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:54:22,334 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:22,888 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:54:23,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:54:23,257 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:54:23,344 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:24,348 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:24,889 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:54:25,317 DEBUG   SenderThread:69178 [sender.py:send():379] send: exit
+2024-12-17 20:54:25,317 INFO    SenderThread:69178 [sender.py:send_exit():586] handling exit code: 0
+2024-12-17 20:54:25,317 INFO    SenderThread:69178 [sender.py:send_exit():588] handling runtime: 227
+2024-12-17 20:54:25,318 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:54:25,318 INFO    SenderThread:69178 [sender.py:send_exit():594] send defer
+2024-12-17 20:54:25,318 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:25,318 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-17 20:54:25,318 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:25,318 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-17 20:54:25,318 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 1
+2024-12-17 20:54:25,318 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:25,318 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-17 20:54:25,319 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:25,319 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-17 20:54:25,319 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 2
+2024-12-17 20:54:25,319 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:25,319 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-17 20:54:25,319 INFO    HandlerThread:69178 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-17 20:54:25,319 INFO    HandlerThread:69178 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-17 20:54:25,319 INFO    HandlerThread:69178 [interfaces.py:finish():200] Joined disk monitor
+2024-12-17 20:54:25,320 DEBUG   SystemMonitor:69178 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-17 20:54:25,320 DEBUG   SystemMonitor:69178 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-17 20:54:25,366 INFO    HandlerThread:69178 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-17 20:54:25,366 INFO    HandlerThread:69178 [interfaces.py:finish():200] Joined memory monitor
+2024-12-17 20:54:25,366 INFO    HandlerThread:69178 [interfaces.py:finish():200] Joined network monitor
+2024-12-17 20:54:25,367 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:25,367 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-17 20:54:25,367 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 3
+2024-12-17 20:54:25,367 DEBUG   SenderThread:69178 [sender.py:send():379] send: stats
+2024-12-17 20:54:25,367 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:25,367 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-17 20:54:25,367 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:25,367 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-17 20:54:25,367 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 4
+2024-12-17 20:54:25,367 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:25,367 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-17 20:54:25,889 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0
+2024-12-17 20:54:25,889 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json
+2024-12-17 20:54:26,148 INFO    wandb-upload_0:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/f5qq7aq2-events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0
+2024-12-17 20:54:26,889 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:54:31,369 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 20:54:31,370 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:54:31,370 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:31,370 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-17 20:54:31,370 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-17 20:54:31,370 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 5
+2024-12-17 20:54:31,370 DEBUG   SenderThread:69178 [sender.py:send():379] send: history
+2024-12-17 20:54:31,370 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:31,370 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:54:31,371 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-17 20:54:31,372 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:54:31,372 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 20:54:31,372 DEBUG   SenderThread:69178 [sender.py:send():379] send: history
+2024-12-17 20:54:31,372 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:54:31,375 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:54:31,376 DEBUG   SenderThread:69178 [sender.py:send():379] send: summary
+2024-12-17 20:54:31,376 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:54:31,376 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:31,376 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-17 20:54:31,376 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 6
+2024-12-17 20:54:31,376 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:31,376 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-17 20:54:31,376 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:31,376 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-17 20:54:31,376 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 7
+2024-12-17 20:54:31,376 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:54:31,377 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:31,377 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-17 20:54:31,377 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:31,377 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-17 20:54:31,891 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json
+2024-12-17 20:54:32,319 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 20:54:33,581 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 8
+2024-12-17 20:54:33,581 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 20:54:33,581 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:33,581 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-17 20:54:33,582 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:33,582 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-17 20:54:33,582 INFO    SenderThread:69178 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-17 20:54:33,584 INFO    SenderThread:69178 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-17 20:54:33,598 INFO    SenderThread:69178 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-17 20:54:33,612 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 9
+2024-12-17 20:54:33,612 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:33,612 DEBUG   SenderThread:69178 [sender.py:send():379] send: artifact
+2024-12-17 20:54:33,612 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-17 20:54:33,892 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:54:34,275 INFO    wandb-upload_3:69178 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0pq2joja
+2024-12-17 20:54:34,275 INFO    wandb-upload_2:69178 [upload_job.py:push():85] Skipped uploading /tmp/tmpb66nr4i8/wandb-job.json
+2024-12-17 20:54:34,320 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 20:54:34,372 INFO    wandb-upload_1:69178 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp9m88tfvz
+2024-12-17 20:54:34,948 INFO    SenderThread:69178 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM4MzA1OTA4Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM4Mjk5MjQ0NQ==', 'versionIndex': 373}}}
+2024-12-17 20:54:34,948 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:34,948 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-17 20:54:34,948 INFO    SenderThread:69178 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-17 20:54:35,893 INFO    SenderThread:69178 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files
+2024-12-17 20:54:35,893 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/config.yaml config.yaml
+2024-12-17 20:54:35,893 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json wandb-summary.json
+2024-12-17 20:54:35,893 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0 events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0
+2024-12-17 20:54:35,893 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/requirements.txt requirements.txt
+2024-12-17 20:54:35,895 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-metadata.json wandb-metadata.json
+2024-12-17 20:54:35,896 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log output.log
+2024-12-17 20:54:35,897 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/diff.patch diff.patch
+2024-12-17 20:54:35,899 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-17 20:54:35,899 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/1_eval_1_3dd750e4041b7d940696.mp4 media/videos/eval/1_eval_1_3dd750e4041b7d940696.mp4
+2024-12-17 20:54:35,899 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/4_eval_4_cb4b291fcf155dd3f366.mp4 media/videos/eval/4_eval_4_cb4b291fcf155dd3f366.mp4
+2024-12-17 20:54:35,899 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/3_eval_3_c5032837d4220bef66c0.mp4 media/videos/eval/3_eval_3_c5032837d4220bef66c0.mp4
+2024-12-17 20:54:35,900 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/2_eval_2_41e781c681092c8d0dae.mp4 media/videos/eval/2_eval_2_41e781c681092c8d0dae.mp4
+2024-12-17 20:54:35,900 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/0_eval_0_479c2ab590e51b63c1e7.mp4 media/videos/eval/0_eval_0_479c2ab590e51b63c1e7.mp4
+2024-12-17 20:54:35,900 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 10
+2024-12-17 20:54:35,900 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 20:54:35,900 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:35,900 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-17 20:54:35,900 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:35,900 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-17 20:54:35,900 INFO    SenderThread:69178 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 20:54:36,085 INFO    wandb-upload_2:69178 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json
+2024-12-17 20:54:36,091 INFO    wandb-upload_3:69178 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/requirements.txt
+2024-12-17 20:54:36,093 INFO    wandb-upload_0:69178 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/config.yaml
+2024-12-17 20:54:36,158 INFO    wandb-upload_1:69178 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:54:36,320 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 20:54:36,321 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 20:54:36,358 INFO    Thread-11 (_thread_body):69178 [sender.py:transition_state():614] send defer: 11
+2024-12-17 20:54:36,358 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:36,359 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-17 20:54:36,359 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:36,359 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-17 20:54:36,359 INFO    SenderThread:69178 [file_pusher.py:join():175] waiting for file pusher
+2024-12-17 20:54:36,360 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 12
+2024-12-17 20:54:36,360 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:36,360 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-17 20:54:36,360 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:36,360 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-17 20:54:36,360 INFO    SenderThread:69178 [file_stream.py:finish():601] file stream finish called
+2024-12-17 20:54:36,567 INFO    SenderThread:69178 [file_stream.py:finish():605] file stream finish is done
+2024-12-17 20:54:36,567 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 13
+2024-12-17 20:54:36,567 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:36,567 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-17 20:54:36,567 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:36,567 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-17 20:54:36,567 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 14
+2024-12-17 20:54:36,567 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:36,567 DEBUG   SenderThread:69178 [sender.py:send():379] send: final
+2024-12-17 20:54:36,568 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-17 20:54:36,568 DEBUG   SenderThread:69178 [sender.py:send():379] send: footer
+2024-12-17 20:54:36,568 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:36,568 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-17 20:54:36,568 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 20:54:36,568 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 20:54:36,568 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: server_info
+2024-12-17 20:54:36,569 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: server_info
+2024-12-17 20:54:36,570 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 20:54:36,570 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-17 20:54:36,570 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-17 20:54:36,571 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:36,626 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 20:54:36,626 INFO    MainThread:69178 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-17 20:54:36,626 INFO    MainThread:69178 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-17 20:54:36,626 INFO    MainThread:69178 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-17 20:54:36,627 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-17 20:54:36,627 INFO    HandlerThread:69178 [handler.py:finish():882] shutting down handler
+2024-12-17 20:54:37,570 INFO    WriterThread:69178 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/run-v5toezqj.wandb
+2024-12-17 20:54:37,626 INFO    SenderThread:69178 [sender.py:finish():1615] shutting down sender
+2024-12-17 20:54:37,626 INFO    SenderThread:69178 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 20:54:37,626 INFO    SenderThread:69178 [file_pusher.py:join():175] waiting for file pusher

205031/wandb/debug.log

@@ -0,0 +1,31 @@
+2024-12-17 20:50:37,554 INFO    MainThread:68994 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-17 20:50:37,554 INFO    MainThread:68994 [wandb_setup.py:_flush():76] Configure stats pid to 68994
+2024-12-17 20:50:37,554 INFO    MainThread:68994 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-17 20:50:37,554 INFO    MainThread:68994 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/settings
+2024-12-17 20:50:37,554 INFO    MainThread:68994 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-17 20:50:37,554 INFO    MainThread:68994 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-17 20:50:37,554 INFO    MainThread:68994 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-17 20:50:37,555 INFO    MainThread:68994 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-17 20:50:37,555 INFO    MainThread:68994 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/logs/debug.log
+2024-12-17 20:50:37,555 INFO    MainThread:68994 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/logs/debug-internal.log
+2024-12-17 20:50:37,555 INFO    MainThread:68994 [wandb_init.py:init():569] calling init triggers
+2024-12-17 20:50:37,555 INFO    MainThread:68994 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '505_0', 'wandb_notes': '505_0_req_577_0', 'eval': True, 'true_action_history': False, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': False, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/505_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031'}
+2024-12-17 20:50:37,555 INFO    MainThread:68994 [wandb_init.py:init():619] starting backend
+2024-12-17 20:50:37,555 INFO    MainThread:68994 [wandb_init.py:init():623] setting up manager
+2024-12-17 20:50:37,560 INFO    MainThread:68994 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-17 20:50:37,560 INFO    MainThread:68994 [wandb_init.py:init():631] backend started and connected
+2024-12-17 20:50:37,572 INFO    MainThread:68994 [wandb_init.py:init():720] updated telemetry
+2024-12-17 20:50:37,578 INFO    MainThread:68994 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-17 20:50:37,830 INFO    MainThread:68994 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-17 20:50:37,904 INFO    MainThread:68994 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-17 20:50:37,904 INFO    MainThread:68994 [wandb_init.py:init():804] starting run threads in backend
+2024-12-17 20:50:38,247 INFO    MainThread:68994 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-17 20:50:38,247 INFO    MainThread:68994 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-17 20:50:38,247 INFO    MainThread:68994 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-17 20:50:38,247 INFO    MainThread:68994 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-17 20:50:38,251 INFO    MainThread:68994 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-17 20:50:38,251 INFO    MainThread:68994 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/tb, True
+2024-12-17 20:50:51,544 INFO    MainThread:68994 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'}
+2024-12-17 20:54:37,640 WARNING MsgRouterThr:68994 [router.py:message_loop():77] message_loop has been closed

205031/wandb/run-20241217_205037-v5toezqj/files/code/FISH/eval_robot.py

@@ -0,0 +1,512 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)		
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()

205031/wandb/run-20241217_205037-v5toezqj/files/config.yaml

@@ -0,0 +1,891 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 0
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '505_0'
+wandb_notes:
+  desc: null
+  value: 505_0_req_577_0
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: false
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: false
+use_sdf_maps:
+  desc: null
+  value: false
+use_normals_maps:
+  desc: null
+  value: false
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value:
+  - hbm_twodim
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 0
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/kostas-graid/datasets/extrinsic_contact_data/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/505_0
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: false
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: false
+        use_sdf_maps: false
+        use_normals_maps: false
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 0
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: false
+          use_sdf_maps: false
+          use_normals_maps: false
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1734486637
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 16
+      - 23
+      - 35
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: hbm
+left_book_slot:
+  desc: null
+  value: twodim

205031/wandb/run-20241217_205037-v5toezqj/files/diff.patch

@@ -0,0 +1,195 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index bb0a413..4b2ad11 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -107,7 +107,7 @@ num_valid_demos: null
+ val_num_groups: 3
+ 
+ # name_of_expert_demo: 'book_insertion_wo_ends_wedge_v2_76_all_demos_10hz_no_filter_wo_flow_mask_expert_demos_imp_act'
+-name_of_expert_demo: '192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act'
++name_of_expert_demo: '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act'
+ expert_dataset_dirpath: '${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}'
+ 
+ expert_dataset: '${expert_dataset_dirpath}/demos.zarr'
+@@ -123,89 +123,28 @@ bc_weight_type: 'qfilter' # linear, qfilter
+ 
+ # Load weights
+ load_checkpoint: ${agent.load_checkpoint}
+-# 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# full 32 demos
+-# wandb_run_id: "265625" # RGBD history seed 0
+-# wandb_run_id: "266370_1" # RGBD history seed 1
+-# wandb_run_id: "266370_2" # RGBD history seed 2
+-# wandb_run_id: "266382_4" # RGBD history seed 4
+-
+-# 16 held out demos (16 demos)
+-# wandb_run_id: "266392_0" # RGBD+act history dataset seed 0
+-# wandb_run_id: "266407_0" # RGBD+act history dataset seed 1
+-wandb_run_id: "266406_0" # RGBD+act history dataset seed 2
+-
+-# wandb_run_id: "265626" # RGBD+Mask+act history seed 0
+-# wandb_run_id: "265638" # RGBD+Mask+act history seed 1
+-# wandb_run_id: "265637" # RGBD+Mask+act history seed 2
+-# wandb_run_id: "265636" # RGBD+Mask+act history seed 3
+-# wandb_run_id: "265635" # RGBD+Mask+act history seed 4
+-
+-# wandb_run_id: "265628" # RGBD+Mask+contact+act history seed 0
+-# wandb_run_id: "265630" # RGBD+Mask+contact+act history seed 1
+-# wandb_run_id: "265631" # RGBD+Mask+contact+act history seed 2
+-# wandb_run_id: "265633" # RGBD+Mask+contact+act history seed 3
+-# wandb_run_id: "265634" # RGBD+Mask+contact+act history seed 4
+-
+-# 8 held out demos (26 demos)
+-# wandb_run_id: "265652" # RGBD+Mask+contact+act history dataset seed 0
+-# wandb_run_id: "265654" # RGBD+Mask+contact+act history dataset seed 1
+-# wandb_run_id: "265655" # RGBD+Mask+contact+act history dataset seed 2
+-
+-# 8 held out demos (26 demos)
+-# wandb_run_id: "265656" # RGBD+Mask+act history dataset seed 0
+-# wandb_run_id: "265657" # RGBD+Mask+act history dataset seed 1
+-# wandb_run_id: "265658" # RGBD+Mask+act history dataset seed 2
+-
+-# 16 held out demos (16 demos)
+-# wandb_run_id: "265641" # RGBD+Mask+act history dataset seed 0
+-# wandb_run_id: "265644" # RGBD+Mask+act history dataset seed 1
+-# wandb_run_id: "265645" # RGBD+Mask+act history dataset seed 2
+-# wandb_run_id: "266381_0" # RGBD+Mask+act history dataset seed 4
+-
+-# 16 held out demos (16 demos)
+-# wandb_run_id: "265646" # RGBD+Mask+Contact+act history dataset seed 0
+-# wandb_run_id: "265649" # RGBD+Mask+Contact+act history dataset seed 1
+-# wandb_run_id: "265650" # RGBD+Mask+Contact+act history dataset seed 2
+-# wandb_run_id: "266378_0" # RGBD+Mask+Contact+act history dataset seed 4
+-
+-
+-
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# full dataset
+-# 6 step action history
+-# wandb_run_id: '250358_0' # RGBD+act history
+-# wandb_run_id: '250371_0' # RGBD+mask+act history
+-# wandb_run_id: '250366_0' # RGBD+contact+act history
+-# wandb_run_id: '250376_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250366_0' # RGBD+contact+act history
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# 16 held out demos
+-# wandb_run_id: '250362_0' # RGBD+act history
+-# wandb_run_id: '250373_0' # RGBD+mask+act history
+-# wandb_run_id: '250378_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250368_0' # RGBD+contact+act history
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# 32 held out demos
+-# wandb_run_id: '250365_0' # RGBD+act history
+-# wandb_run_id: '250375_0' # RGBD+mask+act history
+-wandb_run_id: '250380_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250370_0' # RGBD+contact+act history
+-# 4 step action history
+-# wandb_run_id: '264772_0' # RGBD+act history
+-# wandb_run_id: '264775_0' # RGBD+mask+act history
+-# wandb_run_id: '264776_0' # RGBD+contact+act history
+-# wandb_run_id: '264777_0' # RGBD+mask+contact+act history
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# 16 held out demos
+-# wandb_run_id: '250362_0' # RGBD+act history
+-# wandb_run_id: '250373_0' # RGBD+mask+act history
+-# wandb_run_id: '250378_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250368_0' # RGBD+contact+act history
++# 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
++# 14 demos
++
++# RGBD+act history 
++# wandb_run_id: '497_0' # german
++wandb_run_id: '505_0' # hbm
++# wandb_run_id: '506_0' # lady
++# wandb_run_id: '507_0' # greece
++# wandb_run_id: '508_0' # fowlers
++# wandb_run_id: '509_0' # lib
++# wandb_run_id: '510_0' # modelsys
++# wandb_run_id: '511_0' # electrodyn
++
++# RGBD+mask+act history
++# wandb_run_id: '512_0' # german
++# wandb_run_id: '520_0' # hbm
++# wandb_run_id: '521_0' # lady
++# wandb_run_id: '522_0' # greece
++# wandb_run_id: '523_0' # fowlers
++# wandb_run_id: '524_0' # lib
++# wandb_run_id: '525_0' # modelsys
++# wandb_run_id: '526_0' # electrodyn
+ 
+ # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # 32 held out demos
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index 9406a46..0f225ec 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -15,12 +15,24 @@ if not local_checkpoint_root_dir.exists():
+     local_checkpoint_root_dir.mkdir(parents=True)
+ 
+ # Download the model checkpoints
+-remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1')
++# remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1')
++remote_checkpoint_root_dir = Path('/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1')
+ remote_checkpoint_root_dir = remote_checkpoint_root_dir.expanduser()
+ remote_username = 'leonmkim'
+-remote_host = 'halley'
++remote_host = 'grasp-login1'
+ 
+ #180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
++# run_id_list = [
++#     '512_0', # german
++#     '520_0', # hbm
++#     '521_0', # lady
++#     '522_0', # greece
++#     '523_0', # fowlers
++#     '524_0', # lib
++#     '525_0', # modelsys
++#     '526_0', # electrodyn
++# ]
++
+ # run_id_list = [
+ #     '265638',
+ #     '265637',
+@@ -75,6 +87,17 @@ remote_host = 'halley'
+ # ]
+ 
+ # 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
++# run_id_list = [
++#     '497_0', # german
++#     '505_0', # hbm
++#     '506_0', # lady
++#     '507_0', # greece
++#     '508_0', # fowlers
++#     '509_0', # lib
++#     '510_0', # modelsys
++#     '511_0', # electrodyn
++# ]
++
+ # run_id_list = [
+ #     '266382_4',
+ #     '266370_2',
+@@ -116,13 +139,13 @@ remote_host = 'halley'
+ # ]
+ 
+ # 180x240_1_RGB_D_2.0_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+-run_id_list = [
+-    # '266384_4,# not yet available
+-    '266372_1',
+-    '266372_2',
+-    '265627',
+-    '265620',
+-]
++# run_id_list = [
++#     # '266384_4,# not yet available
++#     '266372_1',
++#     '266372_2',
++#     '265627',
++#     '265620',
++# ]
+ checkpoint_epoch = 12000
+ 
+ # use subprocess to download the model checkpoints in parallel

205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/0_eval_0_479c2ab590e51b63c1e7.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:479c2ab590e51b63c1e7d102b999d930586a97b94c8e0c6eb58a8bb8570849d6
+size 1117369

205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/1_eval_1_3dd750e4041b7d940696.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3dd750e4041b7d94069680d616adf51ded3f6bd3bdb5933c3aac378a2db1dcbe
+size 1120651

205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/2_eval_2_41e781c681092c8d0dae.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41e781c681092c8d0daee8143fbd4dd2873d760169b212b4a595d5eec4a4fa6f
+size 1118222

205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/3_eval_3_c5032837d4220bef66c0.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5032837d4220bef66c08b91bf3175322db59696f748d5d0a44380be57a27b65
+size 1117547

205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/4_eval_4_cb4b291fcf155dd3f366.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb4b291fcf155dd3f3662521ca561d4594281a40887c90201c519e5954662247
+size 1182110

205031/wandb/run-20241217_205037-v5toezqj/files/output.log

@@ -0,0 +1,226 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1734486651.546021]: resetting environment
+[INFO] [1734486651.546989]: cleared current plan
+[INFO] [1734486651.547993]: moving to home
+[INFO] [1734486652.549211]: reached home
+[INFO] [1734486652.550774]: reset action history
+[INFO] [1734486655.261921]: environment reset
+Starting episode 0
+[INFO] [1734486655.265754]: resetting environment
+[INFO] [1734486655.266847]: cleared current plan
+[INFO] [1734486655.268182]: moving to home
+[INFO] [1734486656.269587]: reached home
+[INFO] [1734486656.271345]: reset action history
+[INFO] [1734486658.977686]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1734486664.153425]: resetting environment
+[INFO] [1734486664.154619]: cleared current plan
+[INFO] [1734486664.155332]: moving to home
+[INFO] [1734486665.157266]: reached home
+[INFO] [1734486665.158647]: reset action history
+[INFO] [1734486667.865678]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/episode_rosbags/episode_0_2024-12-17-20-51-07.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-68994.log
+started roslaunch server http://158.130.50.37:39913/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [69300]
+process[print_text-2]: started with pid [69301]
+process[rosbag_record-3]: started with pid [69302]
+started bagging!
+[INFO] [1734486694.649584]: Storing episode...
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit
+[pub_text-1] killing on exit
+[INFO] [1734486695.386858]: Stored episode 1.
+[INFO] [1734486695.397315]: Saving video...
+[INFO] [1734486695.577058]: Video saved!
+Starting episode 1
+[INFO] [1734486695.590763]: resetting environment
+[INFO] [1734486695.591017]: cleared current plan
+[INFO] [1734486695.591225]: moving to home
+[INFO] [1734486699.095536]: reached home
+[INFO] [1734486699.096158]: reset action history
+[INFO] [1734486701.803825]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1734486703.688702]: resetting environment
+[INFO] [1734486703.688999]: cleared current plan
+[INFO] [1734486703.689325]: moving to home
+[INFO] [1734486704.689934]: reached home
+[INFO] [1734486704.690914]: reset action history
+[INFO] [1734486707.397636]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/episode_rosbags/episode_1_2024-12-17-20-51-47.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-68994.log
+started roslaunch server http://158.130.50.37:44151/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [69407]
+process[print_text-5]: started with pid [69408]
+process[rosbag_record-6]: started with pid [69409]
+started bagging!
+[INFO] [1734486734.129622]: Storing episode...
+[rosbag_record-6] killing on exit
+[print_text-5] killing on exit
+[pub_text-4] killing on exit
+[INFO] [1734486734.919917]: Stored episode 2.
+[INFO] [1734486734.920121]: Saving video...
+[INFO] [1734486735.541019]: Video saved!
+Starting episode 2
+[INFO] [1734486735.582081]: resetting environment
+[INFO] [1734486735.583397]: cleared current plan
+[INFO] [1734486735.588396]: moving to home
+[WARN] [1734486735.448673]: Plan exhausted
+[WARN] [1734486735.492759]: Plan exhausted
+[WARN] [1734486735.528848]: Plan exhausted
+[WARN] [1734486735.568192]: Plan exhausted
+[INFO] [1734486739.096748]: reached home
+[INFO] [1734486739.097578]: reset action history
+[INFO] [1734486741.864246]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1734486743.287947]: resetting environment
+[INFO] [1734486743.291170]: cleared current plan
+[INFO] [1734486743.291662]: moving to home
+[INFO] [1734486744.293393]: reached home
+[INFO] [1734486744.294535]: reset action history
+[INFO] [1734486747.002288]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/episode_rosbags/episode_2_2024-12-17-20-52-27.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-68994.log
+started roslaunch server http://158.130.50.37:45359/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [69539]
+process[print_text-8]: started with pid [69540]
+process[rosbag_record-9]: started with pid [69541]
+started bagging!
+[INFO] [1734486773.727993]: Storing episode...
+[rosbag_record-9] killing on exit
+[print_text-8] killing on exit
+[pub_text-7] killing on exit
+[INFO] [1734486774.524223]: Stored episode 3.
+[INFO] [1734486774.525397]: Saving video...
+[INFO] [1734486775.117270]: Video saved!
+Starting episode 3
+[INFO] [1734486775.185832]: resetting environment
+[INFO] [1734486775.191678]: cleared current plan
+[INFO] [1734486775.197125]: moving to home
+[WARN] [1734486775.048221]: Plan exhausted
+[WARN] [1734486775.092434]: Plan exhausted
+[WARN] [1734486775.130260]: Plan exhausted
+[WARN] [1734486775.168456]: Plan exhausted
+[INFO] [1734486778.709794]: reached home
+[INFO] [1734486778.716332]: reset action history
+[INFO] [1734486781.562702]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1734486783.592056]: resetting environment
+[INFO] [1734486783.594496]: cleared current plan
+[INFO] [1734486783.594702]: moving to home
+[INFO] [1734486784.602099]: reached home
+[INFO] [1734486784.612151]: reset action history
+[INFO] [1734486787.354055]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/episode_rosbags/episode_3_2024-12-17-20-53-07.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-68994.log
+started roslaunch server http://158.130.50.37:34253/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-10]: started with pid [69668]
+process[print_text-11]: started with pid [69669]
+process[rosbag_record-12]: started with pid [69670]
+started bagging!
+For topic gripper_width: timestamp difference is -114468299 for nearest: 1734486788719820537 - target: 1734486788834288836 at idx 22
+For topic gripper_width: timestamp difference is -111941731 for nearest: 1734486791919824207 - target: 1734486792031765938 at idx 64
+[INFO] [1734486814.108306]: Storing episode...
+[rosbag_record-12] killing on exit
+[print_text-11] killing on exit[pub_text-10] killing on exit
+[INFO] [1734486814.900166]: Stored episode 4.
+[INFO] [1734486814.905162]: Saving video...
+[INFO] [1734486815.297275]: Video saved!
+Starting episode 4
+[INFO] [1734486815.387428]: resetting environment
+[INFO] [1734486815.397338]: cleared current plan
+[INFO] [1734486815.405188]: moving to home
+[INFO] [1734486818.913381]: reached home
+[INFO] [1734486818.916603]: reset action history
+[INFO] [1734486821.712558]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode![INFO] [1734486823.655343]: resetting environment
+[INFO] [1734486823.659007]: cleared current plan
+[INFO] [1734486823.659798]: moving to home
+[INFO] [1734486824.665421]: reached home
+[INFO] [1734486824.666430]: reset action history
+[INFO] [1734486827.452402]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/episode_rosbags/episode_4_2024-12-17-20-53-47.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-68994.log
+started roslaunch server http://158.130.50.37:45731/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-13]: started with pid [69820]
+process[print_text-14]: started with pid [69821]
+process[rosbag_record-15]: started with pid [69822]
+started bagging!
+For topic gripper_width: timestamp difference is -153719796 for nearest: 1734486836519798623 - target: 1734486836673518419 at idx 130
+For topic gripper_width: timestamp difference is -131065697 for nearest: 1734486847519804865 - target: 1734486847650870562 at idx 249
+[INFO] [1734486855.940370]: Storing episode...
+[rosbag_record-15] killing on exit
+[pub_text-13] killing on exit
+[print_text-14] killing on exit
+[INFO] [1734486856.721085]: Stored episode 5.
+[INFO] [1734486856.731133]: Saving video...
+[INFO] [1734486856.981845]: Video saved!
+[INFO] [1734486857.036105]: resetting environment
+[INFO] [1734486857.046819]: cleared current plan
+[INFO] [1734486857.055888]: moving to home
+[INFO] [1734486860.567082]: reached home
+[INFO] [1734486860.595278]: reset action history
+[INFO] [1734486863.526526]: environment reset
+Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!

205031/wandb/run-20241217_205037-v5toezqj/files/requirements.txt

@@ -0,0 +1,339 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.2
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0

205031/wandb/run-20241217_205037-v5toezqj/files/wandb-metadata.json

@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2024-12-18T01:50:37.929248",
+    "startedAt": "2024-12-18T01:50:37.553164",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "5f28b54373d7d41e4ebace380ff1080a8ffc4051"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 3562.51775,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.999,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.895,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.968,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 360.83281326293945
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}

205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json

@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "479c2ab590e51b63c1e7d102b999d930586a97b94c8e0c6eb58a8bb8570849d6", "size": 1117369, "path": "media/videos/eval/0_eval_0_479c2ab590e51b63c1e7.mp4"}, "global_step": 660, "_timestamp": 1734486865.274416, "_runtime": 227.7131950855255, "_step": 9, "eval/1_eval": {"_type": "video-file", "sha256": "3dd750e4041b7d94069680d616adf51ded3f6bd3bdb5933c3aac378a2db1dcbe", "size": 1120651, "path": "media/videos/eval/1_eval_1_3dd750e4041b7d940696.mp4"}, "eval/num_success": 5.0, "episode": 5.0, "eval/success_rate": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "41e781c681092c8d0daee8143fbd4dd2873d760169b212b4a595d5eec4a4fa6f", "size": 1118222, "path": "media/videos/eval/2_eval_2_41e781c681092c8d0dae.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "c5032837d4220bef66c08b91bf3175322db59696f748d5d0a44380be57a27b65", "size": 1117547, "path": "media/videos/eval/3_eval_3_c5032837d4220bef66c0.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "cb4b291fcf155dd3f3662521ca561d4594281a40887c90201c519e5954662247", "size": 1182110, "path": "media/videos/eval/4_eval_4_cb4b291fcf155dd3f366.mp4"}, "_wandb": {"runtime": 227}}

205031/wandb/run-20241217_205037-v5toezqj/logs/debug-internal.log

@@ -0,0 +1,659 @@
+2024-12-17 20:50:37,561 INFO    StreamThr :69178 [internal.py:wandb_internal():85] W&B internal server running at pid: 69178, started at: 2024-12-17 20:50:37.560794
+2024-12-17 20:50:37,561 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status
+2024-12-17 20:50:37,573 INFO    WriterThread:69178 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/run-v5toezqj.wandb
+2024-12-17 20:50:37,573 DEBUG   SenderThread:69178 [sender.py:send():379] send: header
+2024-12-17 20:50:37,580 DEBUG   SenderThread:69178 [sender.py:send():379] send: run
+2024-12-17 20:50:37,815 INFO    SenderThread:69178 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files
+2024-12-17 20:50:37,815 INFO    SenderThread:69178 [sender.py:_start_run_threads():1188] run started: v5toezqj with start time 1734486637.561221
+2024-12-17 20:50:37,831 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: check_version
+2024-12-17 20:50:37,831 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: check_version
+2024-12-17 20:50:37,909 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: run_start
+2024-12-17 20:50:37,914 DEBUG   HandlerThread:69178 [system_info.py:__init__():26] System info init
+2024-12-17 20:50:37,914 DEBUG   HandlerThread:69178 [system_info.py:__init__():41] System info init done
+2024-12-17 20:50:37,914 INFO    HandlerThread:69178 [system_monitor.py:start():194] Starting system monitor
+2024-12-17 20:50:37,914 INFO    SystemMonitor:69178 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-17 20:50:37,914 INFO    HandlerThread:69178 [system_monitor.py:probe():214] Collecting system info
+2024-12-17 20:50:37,915 INFO    SystemMonitor:69178 [interfaces.py:start():188] Started cpu monitoring
+2024-12-17 20:50:37,915 INFO    SystemMonitor:69178 [interfaces.py:start():188] Started disk monitoring
+2024-12-17 20:50:37,916 INFO    SystemMonitor:69178 [interfaces.py:start():188] Started gpu monitoring
+2024-12-17 20:50:37,916 INFO    SystemMonitor:69178 [interfaces.py:start():188] Started memory monitoring
+2024-12-17 20:50:37,917 INFO    SystemMonitor:69178 [interfaces.py:start():188] Started network monitoring
+2024-12-17 20:50:37,929 DEBUG   HandlerThread:69178 [system_info.py:probe():152] Probing system
+2024-12-17 20:50:37,930 DEBUG   HandlerThread:69178 [system_info.py:_probe_git():137] Probing git
+2024-12-17 20:50:37,935 DEBUG   HandlerThread:69178 [system_info.py:_probe_git():145] Probing git done
+2024-12-17 20:50:37,935 DEBUG   HandlerThread:69178 [system_info.py:probe():200] Probing system done
+2024-12-17 20:50:37,935 DEBUG   HandlerThread:69178 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-18T01:50:37.929248', 'startedAt': '2024-12-18T01:50:37.553164', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '5f28b54373d7d41e4ebace380ff1080a8ffc4051'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.51775, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.895, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.968, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 360.83281326293945}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2024-12-17 20:50:37,935 INFO    HandlerThread:69178 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-17 20:50:37,935 INFO    HandlerThread:69178 [system_monitor.py:probe():227] Publishing system info
+2024-12-17 20:50:37,936 DEBUG   HandlerThread:69178 [system_info.py:_save_code():44] Saving code
+2024-12-17 20:50:37,937 DEBUG   HandlerThread:69178 [system_info.py:_save_code():67] Saving code done
+2024-12-17 20:50:37,937 DEBUG   HandlerThread:69178 [system_info.py:_save_patches():84] Saving git patches
+2024-12-17 20:50:38,004 DEBUG   HandlerThread:69178 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-17 20:50:38,005 INFO    HandlerThread:69178 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-17 20:50:38,008 DEBUG   SenderThread:69178 [sender.py:send():379] send: files
+2024-12-17 20:50:38,008 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-17 20:50:38,008 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-17 20:50:38,008 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-17 20:50:38,246 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-17 20:50:38,246 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: python_packages
+2024-12-17 20:50:38,246 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:50:38,247 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:50:38,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:38,293 INFO    HandlerThread:69178 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/tb"
+  save: true
+}
+control {
+  relay_id: "f5f4f6fa4a0e46c6a826af22cb762514"
+}
+_info {
+  stream_id: "v5toezqj"
+}
+
+2024-12-17 20:50:38,293 DEBUG   HandlerThread:69178 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-17 20:50:38,367 DEBUG   SenderThread:69178 [sender.py:send():379] send: telemetry
+2024-12-17 20:50:38,368 DEBUG   SenderThread:69178 [sender.py:send():379] send: tbrecord
+2024-12-17 20:50:38,368 DEBUG   SenderThread:69178 [sender.py:send():379] send: files
+2024-12-17 20:50:38,368 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0 with policy live
+2024-12-17 20:50:38,386 INFO    wandb-upload_2:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/1louuc8s-diff.patch
+2024-12-17 20:50:38,387 INFO    wandb-upload_0:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/mtmc0peb-wandb-metadata.json
+2024-12-17 20:50:38,387 INFO    wandb-upload_1:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/i4p6vweu-code/FISH/eval_robot.py
+2024-12-17 20:50:38,570 INFO    wandb-upload_3:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/fq8htgz7-events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0
+2024-12-17 20:50:38,817 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/code/FISH/eval_robot.py
+2024-12-17 20:50:38,817 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-metadata.json
+2024-12-17 20:50:38,817 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/requirements.txt
+2024-12-17 20:50:38,818 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/diff.patch
+2024-12-17 20:50:38,818 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:50:38,818 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0
+2024-12-17 20:50:38,818 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/code
+2024-12-17 20:50:38,818 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/code/FISH
+2024-12-17 20:50:39,247 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:40,247 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:40,817 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:50:41,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:42,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:43,247 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:50:43,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:44,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:45,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:46,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:47,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:48,247 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:50:48,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:49,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:50,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:51,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:51,545 DEBUG   SenderThread:69178 [sender.py:send():379] send: config
+2024-12-17 20:50:52,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:52,820 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:50:53,246 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:50:53,247 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:50:53,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:53,369 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:50:54,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:54,820 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:50:55,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:56,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:56,821 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:50:57,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:58,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:50:58,822 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:50:58,978 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:50:59,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:00,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:00,822 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:51:01,250 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:02,250 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:03,251 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:04,153 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:51:04,250 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:05,250 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:06,251 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:06,824 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:51:07,251 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:08,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:51:08,249 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:51:08,293 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:08,824 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:51:09,251 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:09,400 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:51:09,825 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/config.yaml
+2024-12-17 20:51:10,252 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:10,825 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:51:11,252 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:12,253 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:13,253 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:14,253 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:15,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:51:15,253 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:16,253 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:17,253 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:18,253 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:19,253 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:20,248 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:51:20,254 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:21,254 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:22,254 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:23,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:51:23,249 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:51:23,293 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:24,254 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:25,254 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:25,350 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:51:26,254 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:27,254 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:28,255 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:29,255 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:30,255 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:31,250 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:51:31,255 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:32,255 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:33,255 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:34,255 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:35,255 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:35,589 DEBUG   SenderThread:69178 [sender.py:send():379] send: files
+2024-12-17 20:51:35,589 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_479c2ab590e51b63c1e7.mp4 with policy now
+2024-12-17 20:51:35,589 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 20:51:35,590 DEBUG   SenderThread:69178 [sender.py:send():379] send: history
+2024-12-17 20:51:35,590 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:51:35,591 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:51:35,832 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/0_eval_0_479c2ab590e51b63c1e7.mp4
+2024-12-17 20:51:35,832 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json
+2024-12-17 20:51:35,832 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media
+2024-12-17 20:51:35,832 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos
+2024-12-17 20:51:35,832 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval
+2024-12-17 20:51:35,858 INFO    wandb-upload_0:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/aemlia3l-media/videos/eval/0_eval_0_479c2ab590e51b63c1e7.mp4
+2024-12-17 20:51:36,164 INFO    wandb-upload_1:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/370593m2-media/videos/eval/0_eval_0_479c2ab590e51b63c1e7.mp4
+2024-12-17 20:51:36,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:36,592 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:51:36,832 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:51:37,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:37,917 DEBUG   SystemMonitor:69178 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-17 20:51:37,917 DEBUG   SenderThread:69178 [sender.py:send():379] send: stats
+2024-12-17 20:51:38,252 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:51:38,252 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:51:38,293 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:39,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:40,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:40,833 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:51:41,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:41,804 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:51:42,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:42,834 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:51:43,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:43,834 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0
+2024-12-17 20:51:44,122 INFO    wandb-upload_2:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/2lsrgr80-events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0
+2024-12-17 20:51:44,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:44,835 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:51:45,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:46,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:46,835 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:51:47,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:47,398 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:51:48,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:48,836 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:51:49,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:50,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:51,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:52,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:53,249 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:51:53,250 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:51:53,250 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:51:53,293 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:54,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:55,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:56,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:57,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:58,250 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:51:58,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:51:59,258 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:00,258 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:01,258 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:02,258 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:03,250 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:52:03,258 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:04,258 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:05,258 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:06,258 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:07,259 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:07,918 DEBUG   SenderThread:69178 [sender.py:send():379] send: stats
+2024-12-17 20:52:08,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:52:08,257 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:52:08,257 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:52:08,301 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:09,259 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:10,259 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:11,259 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:12,259 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:13,260 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:14,130 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:52:14,260 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:15,262 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:15,570 DEBUG   SenderThread:69178 [sender.py:send():379] send: files
+2024-12-17 20:52:15,570 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_3dd750e4041b7d940696.mp4 with policy now
+2024-12-17 20:52:15,576 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 20:52:15,577 DEBUG   SenderThread:69178 [sender.py:send():379] send: history
+2024-12-17 20:52:15,577 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:52:15,578 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:52:15,832 INFO    wandb-upload_3:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/q6lbjbni-media/videos/eval/1_eval_1_3dd750e4041b7d940696.mp4
+2024-12-17 20:52:15,844 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json
+2024-12-17 20:52:15,845 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/1_eval_1_3dd750e4041b7d940696.mp4
+2024-12-17 20:52:15,845 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval
+2024-12-17 20:52:16,129 INFO    wandb-upload_0:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/inolq7gm-media/videos/eval/1_eval_1_3dd750e4041b7d940696.mp4
+2024-12-17 20:52:16,263 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:16,845 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:52:17,262 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:17,845 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:52:18,263 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:19,137 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:52:19,267 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:20,263 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:20,846 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:52:21,270 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:22,263 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:22,847 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:52:23,251 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:52:23,251 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:52:23,293 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:23,681 DEBUG   SenderThread:69178 [sender.py:send():379] send: history
+2024-12-17 20:52:23,681 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:52:23,703 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:52:23,847 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0
+2024-12-17 20:52:23,847 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json
+2024-12-17 20:52:24,264 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:24,294 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:52:24,848 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:52:25,269 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:26,267 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:26,848 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:52:27,266 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:28,266 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:28,849 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:52:29,266 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:30,254 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:52:30,266 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:31,266 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:32,267 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:33,267 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:34,267 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:35,267 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:36,254 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:52:36,267 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:37,267 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:37,919 DEBUG   SenderThread:69178 [sender.py:send():379] send: stats
+2024-12-17 20:52:38,251 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:52:38,251 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:52:38,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:39,267 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:40,268 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:41,268 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:41,337 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:52:42,268 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:43,268 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:44,268 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:45,268 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:46,268 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:47,255 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:52:47,268 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:48,268 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:49,268 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:50,269 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:51,269 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:52,269 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:53,251 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:52:53,251 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:52:53,293 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:53,398 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:52:54,274 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:54,857 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:52:55,182 DEBUG   SenderThread:69178 [sender.py:send():379] send: files
+2024-12-17 20:52:55,182 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_41e781c681092c8d0dae.mp4 with policy now
+2024-12-17 20:52:55,182 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 20:52:55,183 DEBUG   SenderThread:69178 [sender.py:send():379] send: history
+2024-12-17 20:52:55,183 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:52:55,183 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:52:55,274 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:55,446 INFO    wandb-upload_1:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/rklbn3xb-media/videos/eval/2_eval_2_41e781c681092c8d0dae.mp4
+2024-12-17 20:52:55,858 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json
+2024-12-17 20:52:55,859 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/2_eval_2_41e781c681092c8d0dae.mp4
+2024-12-17 20:52:55,859 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval
+2024-12-17 20:52:56,129 INFO    wandb-upload_2:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/hbutwpli-media/videos/eval/2_eval_2_41e781c681092c8d0dae.mp4
+2024-12-17 20:52:56,281 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:56,859 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:52:57,289 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:57,859 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:52:58,276 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:52:58,711 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:52:59,281 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:00,295 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:00,860 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:01,292 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:02,286 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:02,861 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:03,291 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:03,861 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0
+2024-12-17 20:53:04,105 INFO    wandb-upload_3:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/dvaq4j8i-events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0
+2024-12-17 20:53:04,298 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:04,510 DEBUG   SenderThread:69178 [sender.py:send():379] send: history
+2024-12-17 20:53:04,510 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:53:04,511 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:53:04,511 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:53:04,862 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json
+2024-12-17 20:53:04,862 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:05,293 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:06,295 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:06,862 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:07,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:07,919 DEBUG   SenderThread:69178 [sender.py:send():379] send: stats
+2024-12-17 20:53:08,252 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:53:08,252 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:53:08,294 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:08,863 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:09,296 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:09,937 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:53:10,296 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:10,864 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:11,296 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:12,296 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:13,296 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:14,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:14,865 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:15,255 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:53:15,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:16,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:17,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:18,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:19,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:20,255 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:53:20,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:21,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:22,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:23,252 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:53:23,252 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:53:23,297 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:24,298 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:25,298 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:25,414 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:53:26,298 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:27,298 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:28,298 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:29,298 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:30,298 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:31,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:53:31,298 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:32,298 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:33,299 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:34,299 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:35,301 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:35,372 DEBUG   SenderThread:69178 [sender.py:send():379] send: files
+2024-12-17 20:53:35,372 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_c5032837d4220bef66c0.mp4 with policy now
+2024-12-17 20:53:35,413 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 20:53:35,414 DEBUG   SenderThread:69178 [sender.py:send():379] send: history
+2024-12-17 20:53:35,414 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:53:35,415 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:53:35,629 INFO    wandb-upload_0:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/us394crz-media/videos/eval/3_eval_3_c5032837d4220bef66c0.mp4
+2024-12-17 20:53:35,872 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json
+2024-12-17 20:53:35,872 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/3_eval_3_c5032837d4220bef66c0.mp4
+2024-12-17 20:53:35,872 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval
+2024-12-17 20:53:36,136 INFO    wandb-upload_1:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/c8thr94g-media/videos/eval/3_eval_3_c5032837d4220bef66c0.mp4
+2024-12-17 20:53:36,309 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:36,415 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:53:36,872 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:37,306 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:37,919 DEBUG   SenderThread:69178 [sender.py:send():379] send: stats
+2024-12-17 20:53:38,269 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:53:38,269 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:53:38,352 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:39,316 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:40,318 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:40,874 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:41,330 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:41,718 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:53:42,331 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:42,874 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:43,328 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:43,875 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0
+2024-12-17 20:53:44,327 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:44,557 DEBUG   SenderThread:69178 [sender.py:send():379] send: history
+2024-12-17 20:53:44,557 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:53:44,558 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:53:44,875 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json
+2024-12-17 20:53:44,875 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:45,319 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:46,319 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:46,876 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:47,327 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:47,452 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:53:48,320 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:48,876 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:49,341 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:50,321 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:50,877 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:51,321 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:52,321 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:53,252 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:53:53,252 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:53:53,322 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:53,406 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:53:54,321 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:55,322 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:56,322 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:57,322 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:58,322 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:53:58,880 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:53:59,258 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:53:59,322 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:00,322 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:01,322 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:02,322 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:03,323 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:04,323 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:05,258 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:54:05,323 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:06,323 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:07,323 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:07,920 DEBUG   SenderThread:69178 [sender.py:send():379] send: stats
+2024-12-17 20:54:08,252 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:54:08,252 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:54:08,324 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:08,883 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:54:09,324 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:10,324 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:10,391 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:54:11,324 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:12,324 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:13,324 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:14,324 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:15,324 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:15,940 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:54:16,325 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:16,886 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:54:17,032 DEBUG   SenderThread:69178 [sender.py:send():379] send: files
+2024-12-17 20:54:17,032 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_cb4b291fcf155dd3f366.mp4 with policy now
+2024-12-17 20:54:17,035 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 20:54:17,035 DEBUG   SenderThread:69178 [sender.py:send():379] send: history
+2024-12-17 20:54:17,035 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:54:17,036 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:54:17,297 INFO    wandb-upload_2:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/07e8vmtr-media/videos/eval/4_eval_4_cb4b291fcf155dd3f366.mp4
+2024-12-17 20:54:17,331 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:17,886 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json
+2024-12-17 20:54:17,886 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/4_eval_4_cb4b291fcf155dd3f366.mp4
+2024-12-17 20:54:17,887 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval
+2024-12-17 20:54:18,176 INFO    wandb-upload_3:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/c23rhxtl-media/videos/eval/4_eval_4_cb4b291fcf155dd3f366.mp4
+2024-12-17 20:54:18,336 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:18,887 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:54:19,335 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:20,330 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:21,335 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:21,617 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:54:22,334 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:22,888 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:54:23,256 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:54:23,257 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:54:23,344 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:24,348 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:24,889 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:54:25,317 DEBUG   SenderThread:69178 [sender.py:send():379] send: exit
+2024-12-17 20:54:25,317 INFO    SenderThread:69178 [sender.py:send_exit():586] handling exit code: 0
+2024-12-17 20:54:25,317 INFO    SenderThread:69178 [sender.py:send_exit():588] handling runtime: 227
+2024-12-17 20:54:25,318 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:54:25,318 INFO    SenderThread:69178 [sender.py:send_exit():594] send defer
+2024-12-17 20:54:25,318 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:25,318 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-17 20:54:25,318 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:25,318 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-17 20:54:25,318 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 1
+2024-12-17 20:54:25,318 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:25,318 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-17 20:54:25,319 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:25,319 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-17 20:54:25,319 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 2
+2024-12-17 20:54:25,319 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:25,319 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-17 20:54:25,319 INFO    HandlerThread:69178 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-17 20:54:25,319 INFO    HandlerThread:69178 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-17 20:54:25,319 INFO    HandlerThread:69178 [interfaces.py:finish():200] Joined disk monitor
+2024-12-17 20:54:25,320 DEBUG   SystemMonitor:69178 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-17 20:54:25,320 DEBUG   SystemMonitor:69178 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-17 20:54:25,366 INFO    HandlerThread:69178 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-17 20:54:25,366 INFO    HandlerThread:69178 [interfaces.py:finish():200] Joined memory monitor
+2024-12-17 20:54:25,366 INFO    HandlerThread:69178 [interfaces.py:finish():200] Joined network monitor
+2024-12-17 20:54:25,367 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:25,367 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-17 20:54:25,367 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 3
+2024-12-17 20:54:25,367 DEBUG   SenderThread:69178 [sender.py:send():379] send: stats
+2024-12-17 20:54:25,367 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:25,367 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-17 20:54:25,367 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:25,367 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-17 20:54:25,367 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 4
+2024-12-17 20:54:25,367 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:25,367 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-17 20:54:25,889 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0
+2024-12-17 20:54:25,889 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json
+2024-12-17 20:54:26,148 INFO    wandb-upload_0:69178 [upload_job.py:push():130] Uploaded file /tmp/tmphf0ydbrfwandb/f5qq7aq2-events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0
+2024-12-17 20:54:26,889 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:54:31,369 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 20:54:31,370 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:54:31,370 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:31,370 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-17 20:54:31,370 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-17 20:54:31,370 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 5
+2024-12-17 20:54:31,370 DEBUG   SenderThread:69178 [sender.py:send():379] send: history
+2024-12-17 20:54:31,370 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:31,370 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:54:31,371 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-17 20:54:31,372 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:54:31,372 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 20:54:31,372 DEBUG   SenderThread:69178 [sender.py:send():379] send: history
+2024-12-17 20:54:31,372 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:54:31,375 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:54:31,376 DEBUG   SenderThread:69178 [sender.py:send():379] send: summary
+2024-12-17 20:54:31,376 INFO    SenderThread:69178 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:54:31,376 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:31,376 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-17 20:54:31,376 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 6
+2024-12-17 20:54:31,376 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:31,376 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-17 20:54:31,376 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:31,376 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-17 20:54:31,376 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 7
+2024-12-17 20:54:31,376 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:54:31,377 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:31,377 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-17 20:54:31,377 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:31,377 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-17 20:54:31,891 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json
+2024-12-17 20:54:32,319 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 20:54:33,581 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 8
+2024-12-17 20:54:33,581 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 20:54:33,581 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:33,581 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-17 20:54:33,582 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:33,582 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-17 20:54:33,582 INFO    SenderThread:69178 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-17 20:54:33,584 INFO    SenderThread:69178 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-17 20:54:33,598 INFO    SenderThread:69178 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-17 20:54:33,612 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 9
+2024-12-17 20:54:33,612 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:33,612 DEBUG   SenderThread:69178 [sender.py:send():379] send: artifact
+2024-12-17 20:54:33,612 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-17 20:54:33,892 INFO    Thread-12 :69178 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:54:34,275 INFO    wandb-upload_3:69178 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0pq2joja
+2024-12-17 20:54:34,275 INFO    wandb-upload_2:69178 [upload_job.py:push():85] Skipped uploading /tmp/tmpb66nr4i8/wandb-job.json
+2024-12-17 20:54:34,320 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 20:54:34,372 INFO    wandb-upload_1:69178 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp9m88tfvz
+2024-12-17 20:54:34,948 INFO    SenderThread:69178 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM4MzA1OTA4Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM4Mjk5MjQ0NQ==', 'versionIndex': 373}}}
+2024-12-17 20:54:34,948 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:34,948 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-17 20:54:34,948 INFO    SenderThread:69178 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-17 20:54:35,893 INFO    SenderThread:69178 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files
+2024-12-17 20:54:35,893 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/config.yaml config.yaml
+2024-12-17 20:54:35,893 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json wandb-summary.json
+2024-12-17 20:54:35,893 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0 events.out.tfevents.1734486638.leonmkim-ROG-Strix-G15CS-G15CS.68994.0
+2024-12-17 20:54:35,893 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/requirements.txt requirements.txt
+2024-12-17 20:54:35,895 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-metadata.json wandb-metadata.json
+2024-12-17 20:54:35,896 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log output.log
+2024-12-17 20:54:35,897 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/diff.patch diff.patch
+2024-12-17 20:54:35,899 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-17 20:54:35,899 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/1_eval_1_3dd750e4041b7d940696.mp4 media/videos/eval/1_eval_1_3dd750e4041b7d940696.mp4
+2024-12-17 20:54:35,899 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/4_eval_4_cb4b291fcf155dd3f366.mp4 media/videos/eval/4_eval_4_cb4b291fcf155dd3f366.mp4
+2024-12-17 20:54:35,899 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/3_eval_3_c5032837d4220bef66c0.mp4 media/videos/eval/3_eval_3_c5032837d4220bef66c0.mp4
+2024-12-17 20:54:35,900 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/2_eval_2_41e781c681092c8d0dae.mp4 media/videos/eval/2_eval_2_41e781c681092c8d0dae.mp4
+2024-12-17 20:54:35,900 INFO    SenderThread:69178 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/media/videos/eval/0_eval_0_479c2ab590e51b63c1e7.mp4 media/videos/eval/0_eval_0_479c2ab590e51b63c1e7.mp4
+2024-12-17 20:54:35,900 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 10
+2024-12-17 20:54:35,900 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 20:54:35,900 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:35,900 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-17 20:54:35,900 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:35,900 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-17 20:54:35,900 INFO    SenderThread:69178 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 20:54:36,085 INFO    wandb-upload_2:69178 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/wandb-summary.json
+2024-12-17 20:54:36,091 INFO    wandb-upload_3:69178 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/requirements.txt
+2024-12-17 20:54:36,093 INFO    wandb-upload_0:69178 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/config.yaml
+2024-12-17 20:54:36,158 INFO    wandb-upload_1:69178 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/files/output.log
+2024-12-17 20:54:36,320 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 20:54:36,321 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 20:54:36,358 INFO    Thread-11 (_thread_body):69178 [sender.py:transition_state():614] send defer: 11
+2024-12-17 20:54:36,358 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:36,359 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-17 20:54:36,359 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:36,359 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-17 20:54:36,359 INFO    SenderThread:69178 [file_pusher.py:join():175] waiting for file pusher
+2024-12-17 20:54:36,360 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 12
+2024-12-17 20:54:36,360 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:36,360 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-17 20:54:36,360 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:36,360 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-17 20:54:36,360 INFO    SenderThread:69178 [file_stream.py:finish():601] file stream finish called
+2024-12-17 20:54:36,567 INFO    SenderThread:69178 [file_stream.py:finish():605] file stream finish is done
+2024-12-17 20:54:36,567 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 13
+2024-12-17 20:54:36,567 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:36,567 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-17 20:54:36,567 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:36,567 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-17 20:54:36,567 INFO    SenderThread:69178 [sender.py:transition_state():614] send defer: 14
+2024-12-17 20:54:36,567 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 20:54:36,567 DEBUG   SenderThread:69178 [sender.py:send():379] send: final
+2024-12-17 20:54:36,568 INFO    HandlerThread:69178 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-17 20:54:36,568 DEBUG   SenderThread:69178 [sender.py:send():379] send: footer
+2024-12-17 20:54:36,568 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: defer
+2024-12-17 20:54:36,568 INFO    SenderThread:69178 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-17 20:54:36,568 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 20:54:36,568 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 20:54:36,568 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: server_info
+2024-12-17 20:54:36,569 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: server_info
+2024-12-17 20:54:36,570 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 20:54:36,570 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-17 20:54:36,570 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-17 20:54:36,571 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:54:36,626 DEBUG   SenderThread:69178 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 20:54:36,626 INFO    MainThread:69178 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-17 20:54:36,626 INFO    MainThread:69178 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-17 20:54:36,626 INFO    MainThread:69178 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-17 20:54:36,627 DEBUG   HandlerThread:69178 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-17 20:54:36,627 INFO    HandlerThread:69178 [handler.py:finish():882] shutting down handler
+2024-12-17 20:54:37,570 INFO    WriterThread:69178 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/run-v5toezqj.wandb
+2024-12-17 20:54:37,626 INFO    SenderThread:69178 [sender.py:finish():1615] shutting down sender
+2024-12-17 20:54:37,626 INFO    SenderThread:69178 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 20:54:37,626 INFO    SenderThread:69178 [file_pusher.py:join():175] waiting for file pusher

205031/wandb/run-20241217_205037-v5toezqj/logs/debug.log

@@ -0,0 +1,31 @@
+2024-12-17 20:50:37,554 INFO    MainThread:68994 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-17 20:50:37,554 INFO    MainThread:68994 [wandb_setup.py:_flush():76] Configure stats pid to 68994
+2024-12-17 20:50:37,554 INFO    MainThread:68994 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-17 20:50:37,554 INFO    MainThread:68994 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/settings
+2024-12-17 20:50:37,554 INFO    MainThread:68994 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-17 20:50:37,554 INFO    MainThread:68994 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-17 20:50:37,554 INFO    MainThread:68994 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-17 20:50:37,555 INFO    MainThread:68994 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-17 20:50:37,555 INFO    MainThread:68994 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/logs/debug.log
+2024-12-17 20:50:37,555 INFO    MainThread:68994 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/wandb/run-20241217_205037-v5toezqj/logs/debug-internal.log
+2024-12-17 20:50:37,555 INFO    MainThread:68994 [wandb_init.py:init():569] calling init triggers
+2024-12-17 20:50:37,555 INFO    MainThread:68994 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '505_0', 'wandb_notes': '505_0_req_577_0', 'eval': True, 'true_action_history': False, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': False, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/505_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031'}
+2024-12-17 20:50:37,555 INFO    MainThread:68994 [wandb_init.py:init():619] starting backend
+2024-12-17 20:50:37,555 INFO    MainThread:68994 [wandb_init.py:init():623] setting up manager
+2024-12-17 20:50:37,560 INFO    MainThread:68994 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-17 20:50:37,560 INFO    MainThread:68994 [wandb_init.py:init():631] backend started and connected
+2024-12-17 20:50:37,572 INFO    MainThread:68994 [wandb_init.py:init():720] updated telemetry
+2024-12-17 20:50:37,578 INFO    MainThread:68994 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-17 20:50:37,830 INFO    MainThread:68994 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-17 20:50:37,904 INFO    MainThread:68994 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-17 20:50:37,904 INFO    MainThread:68994 [wandb_init.py:init():804] starting run threads in backend
+2024-12-17 20:50:38,247 INFO    MainThread:68994 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-17 20:50:38,247 INFO    MainThread:68994 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-17 20:50:38,247 INFO    MainThread:68994 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-17 20:50:38,247 INFO    MainThread:68994 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-17 20:50:38,251 INFO    MainThread:68994 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-17 20:50:38,251 INFO    MainThread:68994 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/505_0/205031/tb, True
+2024-12-17 20:50:51,544 INFO    MainThread:68994 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'}
+2024-12-17 20:54:37,640 WARNING MsgRouterThr:68994 [router.py:message_loop():77] message_loop has been closed

config.yaml

@@ -0,0 +1,546 @@
+root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data
+replay_buffer_size: 150000
+replay_buffer_num_workers: 2
+nstep: 3
+batch_size: 128
+seed: 0
+dataset_shuffle_seed: 0
+device: cuda
+save_video: true
+save_train_video: true
+use_tb: true
+use_wandb: true
+wandb_run_id: '505_0'
+wandb_notes: 505_0_req_577_0
+eval: false
+true_action_history: false
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.2
+env_dtc_max_value: 0.4
+grasped_normals_mask_max_dtc_value: 0.2
+env_normals_mask_max_dtc_value: 0.4
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: true
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type: small
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+baseline: false
+train_demo_idxs_list_or_num: -1
+log_train_every_steps: 25
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+store_dataset_in_memory: false
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+include_groups_list:
+- hbm_twodim
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+name_of_valid_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr
+valid_demo_idxs_list_or_num: null
+val_num_groups: 0
+load_bc: ${agent.load_checkpoint}
+checkpoint_epoch_list:
+- 99
+- 199
+- 299
+- 399
+- 499
+- 599
+- 699
+- 799
+- 899
+- 999
+- 1249
+- 1499
+- 1749
+- 1999
+- 2999
+- 3999
+- 4999
+- 5999
+- 6999
+- 7999
+- 8999
+- 9999
+snapshot_root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data/FISH
+save_snapshot: true
+save_last_snapshot: true
+save_snapshot_when_done: true
+top_k_checkpoints: 5
+save_snapshot_link_to_weights_dir: deprecated
+bc_regularize: false
+bc_weight_type: qfilter
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: 6
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+      input_shapes:
+        observation.image:
+        - 13
+        - 180
+        - 240
+        context_observation.image:
+        - 13
+        - 180
+        - 240
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: cosine
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: 0.0001
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 1000000
+  num_seed_frames: 0
+  num_train_epochs: 15000
+  validate_every_epochs: 250
+  validate_diffusion_on_action_loss_every_epochs: 250
+  train_eval_diffusion_on_action_loss_every_epochs: 250
+  check_topk_every_epochs: 250
+  save_snapshot_every_epochs: 1500
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: ${device}
+      cam_resize_shape: ${agent.desired_image_shape}
+      orig_cam_shape: ${agent.orig_cam_shape}
+      policy_frequency: ${policy_frequency}
+      interpolation_frequency: ${interpolation_frequency}
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: ${agent.config.policy_cfg.horizon}
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: ${agent.config.policy_cfg.kernel_size}
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: ${num_train_frames_diffusion}
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: ${use_depth}
+        use_color: ${use_color}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_masks}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${color_crop_type}
+          depth_crop_type: ${depth_crop_type}
+          segmask_crop_type: ${segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${add_crop_binary_mask}
+          add_coord_conv_map: ${add_coord_conv_map}
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: ${use_context_color}
+          use_depth: ${use_context_depth}
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: ${use_context_segmask}
+            representation: ${mask_representation}
+            mask_list: ${mask_list}
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: ${context_color_crop_type}
+            depth_crop_type: ${context_depth_crop_type}
+            segmask_crop_type: ${context_segmask_crop_type}
+            crop_hw: ${crop_hw}
+            crop_down_offset: ${crop_down_offset}
+            add_crop_binary_mask: ${context_add_crop_binary_mask}
+            add_coord_conv_map: ${context_add_coord_conv_map}
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: ${use_contact_map}
+        use_sdf_maps: ${use_sdf_maps}
+        use_normals_maps: ${use_normals_maps}
+        which_objects: ${which_objects}
+        grasped_dtc_max_value: ${grasped_dtc_max_value}
+        env_dtc_max_value: ${env_dtc_max_value}
+        grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+        env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+        clamp_dtc: ${clamp_dtc}
+        max_contact_prob: ${max_contact_prob}
+        mask_normals_within_sdf: ${mask_normals_within_sdf}
+        dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+        adaptive_normals_mask: ${adaptive_normals_mask}
+        max_depth: ${max_depth}
+        image_shape: ${agent.desired_image_shape}
+        learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: ${contact_model_name}
+        zero_centered: false
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 15000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 250
+validate_diffusion_on_action_loss_every_epochs: 250
+train_eval_diffusion_on_action_loss_every_epochs: 250
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: 250
+save_snapshot_every_epochs_diffusion: 1500
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1