Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,18 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+210157/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/0_eval_0_c14cbfe41183ade1bc4f.mp4 filter=lfs diff=lfs merge=lfs -text
+210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/1_eval_1_a2d5dd163e6dc31055f4.mp4 filter=lfs diff=lfs merge=lfs -text
+210157/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+210157/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+210157/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/3_eval_3_a5ebabbd104fafab5ecd.mp4 filter=lfs diff=lfs merge=lfs -text
+210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/2_eval_2_c3a9936bb29a61a5d383.mp4 filter=lfs diff=lfs merge=lfs -text
+210157/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/4_eval_4_a10c49f08e3e38202133.mp4 filter=lfs diff=lfs merge=lfs -text
+210157/episode_rosbags/episode_2_2024-12-17-21-03-47.bag filter=lfs diff=lfs merge=lfs -text
+210157/episode_rosbags/episode_0_2024-12-17-21-02-27.bag filter=lfs diff=lfs merge=lfs -text
+210157/episode_rosbags/episode_1_2024-12-17-21-03-06.bag filter=lfs diff=lfs merge=lfs -text
+210157/episode_rosbags/episode_3_2024-12-17-21-04-30.bag filter=lfs diff=lfs merge=lfs -text
+210157/episode_rosbags/episode_4_2024-12-17-21-05-10.bag filter=lfs diff=lfs merge=lfs -text

210157/.hydra/config.yaml

@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '507_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${now:%H%M%S}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

210157/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

210157/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion

210157/episode_rosbags/aligned_depth_to_color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

210157/episode_rosbags/cam_tf_world.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256

210157/episode_rosbags/color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

210157/episode_rosbags/depth_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200

210157/episode_rosbags/episode_0_2024-12-17-21-02-27.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59cfca0969f1ea1008cec81a85addb134a537c895d7ba493b2c255ed98f2b2b4
+size 1334461973

210157/episode_rosbags/episode_1_2024-12-17-21-03-06.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7b0b2bdb9ee4f86f3360e236222ba9167bb5a2d8671b7cd77e2e8d418ca741f
+size 1331764610

210157/episode_rosbags/episode_2_2024-12-17-21-03-47.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b5ebd97e13b57d990b71afe158cd309c23dae7cff7c8809d8dc6be4788e094b
+size 1335805137

210157/episode_rosbags/episode_3_2024-12-17-21-04-30.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d59f7de2f28c06cb5a2179398f57c3203f4e7b4246e8362936629b2de3330637
+size 1332225228

210157/episode_rosbags/episode_4_2024-12-17-21-05-10.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e09ec670b33211c31fe826b840f2595a6c498396974f25da8d49670f2ad8897
+size 1334746535

210157/eval_policy.log

@@ -0,0 +1,15 @@
+[2024-12-17 21:01:57,164][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-17 21:01:57,167][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2024-12-17 21:01:59,246][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+

210157/eval_robot.log

@@ -0,0 +1,12 @@
+[2024-12-17 21:01:57,909][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-17 21:01:57,912][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+

210157/eval_video/0_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c14cbfe41183ade1bc4ff61f35119dcc66addcd0c91751b9973014f71084eb34
+size 1137780

210157/eval_video/1_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2d5dd163e6dc31055f490b0a07dc67d943975e1db66596df8c9907747fff5dc
+size 1132118

210157/eval_video/2_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3a9936bb29a61a5d383a2ded237402015093b53dcab382212c28520b1b10874
+size 1135489

210157/eval_video/3_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5ebabbd104fafab5ecd21d62ee8f1c213976ba38e174b9c18f59177a29e256b
+size 1145514

210157/eval_video/4_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a10c49f08e3e38202133932429d20d6b9990c27d3c533d8f6dade92cf1d8175c
+size 1133992

210157/tb/events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4995d23f3e95fc62a875375f0f9784700ab42fcc7580156cdcb231021be2e369
+size 1103

210157/wandb/debug-internal.log

@@ -0,0 +1,654 @@
+2024-12-17 21:02:03,780 INFO    StreamThr :72042 [internal.py:wandb_internal():85] W&B internal server running at pid: 72042, started at: 2024-12-17 21:02:03.780301
+2024-12-17 21:02:03,781 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status
+2024-12-17 21:02:03,792 INFO    WriterThread:72042 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/run-tbm1o8qf.wandb
+2024-12-17 21:02:03,793 DEBUG   SenderThread:72042 [sender.py:send():379] send: header
+2024-12-17 21:02:03,801 DEBUG   SenderThread:72042 [sender.py:send():379] send: run
+2024-12-17 21:02:04,040 INFO    SenderThread:72042 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files
+2024-12-17 21:02:04,040 INFO    SenderThread:72042 [sender.py:_start_run_threads():1188] run started: tbm1o8qf with start time 1734487323.781326
+2024-12-17 21:02:04,055 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: check_version
+2024-12-17 21:02:04,055 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: check_version
+2024-12-17 21:02:04,101 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: run_start
+2024-12-17 21:02:04,105 DEBUG   HandlerThread:72042 [system_info.py:__init__():26] System info init
+2024-12-17 21:02:04,105 DEBUG   HandlerThread:72042 [system_info.py:__init__():41] System info init done
+2024-12-17 21:02:04,105 INFO    HandlerThread:72042 [system_monitor.py:start():194] Starting system monitor
+2024-12-17 21:02:04,105 INFO    SystemMonitor:72042 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-17 21:02:04,106 INFO    HandlerThread:72042 [system_monitor.py:probe():214] Collecting system info
+2024-12-17 21:02:04,106 INFO    SystemMonitor:72042 [interfaces.py:start():188] Started cpu monitoring
+2024-12-17 21:02:04,106 INFO    SystemMonitor:72042 [interfaces.py:start():188] Started disk monitoring
+2024-12-17 21:02:04,107 INFO    SystemMonitor:72042 [interfaces.py:start():188] Started gpu monitoring
+2024-12-17 21:02:04,107 INFO    SystemMonitor:72042 [interfaces.py:start():188] Started memory monitoring
+2024-12-17 21:02:04,108 INFO    SystemMonitor:72042 [interfaces.py:start():188] Started network monitoring
+2024-12-17 21:02:04,118 DEBUG   HandlerThread:72042 [system_info.py:probe():152] Probing system
+2024-12-17 21:02:04,120 DEBUG   HandlerThread:72042 [system_info.py:_probe_git():137] Probing git
+2024-12-17 21:02:04,124 DEBUG   HandlerThread:72042 [system_info.py:_probe_git():145] Probing git done
+2024-12-17 21:02:04,125 DEBUG   HandlerThread:72042 [system_info.py:probe():200] Probing system done
+2024-12-17 21:02:04,125 DEBUG   HandlerThread:72042 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-18T02:02:04.118743', 'startedAt': '2024-12-18T02:02:03.773730', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '5f28b54373d7d41e4ebace380ff1080a8ffc4051'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.995375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.94, 'min': 800.0, 'max': 4700.0}, {'current': 4500.013, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 373.2729034423828}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2024-12-17 21:02:04,125 INFO    HandlerThread:72042 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-17 21:02:04,125 INFO    HandlerThread:72042 [system_monitor.py:probe():227] Publishing system info
+2024-12-17 21:02:04,125 DEBUG   HandlerThread:72042 [system_info.py:_save_code():44] Saving code
+2024-12-17 21:02:04,126 DEBUG   HandlerThread:72042 [system_info.py:_save_code():67] Saving code done
+2024-12-17 21:02:04,126 DEBUG   HandlerThread:72042 [system_info.py:_save_patches():84] Saving git patches
+2024-12-17 21:02:04,191 DEBUG   HandlerThread:72042 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-17 21:02:04,191 INFO    HandlerThread:72042 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-17 21:02:04,194 DEBUG   SenderThread:72042 [sender.py:send():379] send: files
+2024-12-17 21:02:04,195 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-17 21:02:04,195 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-17 21:02:04,195 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-17 21:02:04,435 INFO    wandb-upload_0:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/5f5971ts-wandb-metadata.json
+2024-12-17 21:02:04,439 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-17 21:02:04,439 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: python_packages
+2024-12-17 21:02:04,440 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:02:04,440 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:02:04,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:04,444 INFO    HandlerThread:72042 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/tb"
+  save: true
+}
+control {
+  relay_id: "4e4e9038b20c4fde8df6b85c0bf5638d"
+}
+_info {
+  stream_id: "tbm1o8qf"
+}
+
+2024-12-17 21:02:04,444 DEBUG   HandlerThread:72042 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-17 21:02:04,488 INFO    wandb-upload_2:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/4v1c1pqd-diff.patch
+2024-12-17 21:02:04,499 INFO    wandb-upload_1:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/k6qe8mxm-code/FISH/eval_robot.py
+2024-12-17 21:02:04,542 DEBUG   SenderThread:72042 [sender.py:send():379] send: telemetry
+2024-12-17 21:02:04,542 DEBUG   SenderThread:72042 [sender.py:send():379] send: tbrecord
+2024-12-17 21:02:04,543 DEBUG   SenderThread:72042 [sender.py:send():379] send: files
+2024-12-17 21:02:04,543 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0 with policy live
+2024-12-17 21:02:04,725 INFO    wandb-upload_0:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/fvv96vo3-events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0
+2024-12-17 21:02:05,042 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:05,042 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/requirements.txt
+2024-12-17 21:02:05,042 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-metadata.json
+2024-12-17 21:02:05,042 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/code/FISH/eval_robot.py
+2024-12-17 21:02:05,042 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/diff.patch
+2024-12-17 21:02:05,042 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0
+2024-12-17 21:02:05,043 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/code/FISH
+2024-12-17 21:02:05,043 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/code
+2024-12-17 21:02:05,440 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:06,440 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:07,041 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:07,440 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:08,440 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:09,440 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:02:09,441 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:10,441 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:11,441 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:11,984 DEBUG   SenderThread:72042 [sender.py:send():379] send: config
+2024-12-17 21:02:12,441 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:13,043 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:13,442 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:14,441 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:02:14,442 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:15,043 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:15,442 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:16,442 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:17,044 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:17,442 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:18,442 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:19,045 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:19,461 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:02:19,461 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:19,462 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:02:19,463 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:02:20,442 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:21,045 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:21,442 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:22,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:23,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:24,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:24,944 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:02:25,046 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:25,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:26,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:27,047 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:27,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:28,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:29,047 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:29,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:30,441 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:02:30,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:31,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:32,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:33,049 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:33,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:34,440 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:02:34,440 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:02:34,481 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:35,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:35,585 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:02:36,049 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/config.yaml
+2024-12-17 21:02:36,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:37,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:38,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:39,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:40,445 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:41,441 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:02:41,445 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:42,445 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:43,445 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:44,445 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:45,445 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:46,441 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:02:46,445 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:47,445 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:48,445 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:49,440 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:02:49,440 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:02:49,481 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:50,446 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:51,447 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:51,579 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:02:52,447 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:53,447 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:54,447 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:55,404 DEBUG   SenderThread:72042 [sender.py:send():379] send: files
+2024-12-17 21:02:55,404 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c14cbfe41183ade1bc4f.mp4 with policy now
+2024-12-17 21:02:55,405 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:02:55,405 DEBUG   SenderThread:72042 [sender.py:send():379] send: history
+2024-12-17 21:02:55,406 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:02:55,407 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:02:55,448 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:55,658 INFO    wandb-upload_1:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/ft8yc6fq-media/videos/eval/0_eval_0_c14cbfe41183ade1bc4f.mp4
+2024-12-17 21:02:56,055 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json
+2024-12-17 21:02:56,055 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/0_eval_0_c14cbfe41183ade1bc4f.mp4
+2024-12-17 21:02:56,055 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos
+2024-12-17 21:02:56,055 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media
+2024-12-17 21:02:56,055 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval
+2024-12-17 21:02:56,311 INFO    wandb-upload_0:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/0udxobya-media/videos/eval/0_eval_0_c14cbfe41183ade1bc4f.mp4
+2024-12-17 21:02:56,447 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:57,055 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:57,408 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:02:57,448 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:58,453 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:59,448 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:00,448 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:01,056 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:01,448 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:02,449 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:02,607 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:03:03,057 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:03,057 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0
+2024-12-17 21:03:03,449 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:04,108 DEBUG   SystemMonitor:72042 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-17 21:03:04,109 DEBUG   SenderThread:72042 [sender.py:send():379] send: stats
+2024-12-17 21:03:04,441 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:03:04,441 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:03:04,449 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:05,058 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:05,450 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:06,450 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:07,450 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:08,450 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:08,549 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:03:09,059 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:09,450 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:10,450 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:11,450 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:12,451 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:13,451 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:14,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:03:14,452 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:15,452 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:16,452 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:17,452 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:18,452 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:19,440 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:03:19,440 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:03:19,481 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:19,578 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:03:20,452 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:21,454 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:22,455 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:23,455 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:24,455 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:25,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:03:25,455 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:26,455 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:27,456 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:28,456 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:29,456 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:30,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:03:30,457 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:31,457 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:32,457 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:33,457 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:34,109 DEBUG   SenderThread:72042 [sender.py:send():379] send: stats
+2024-12-17 21:03:34,382 DEBUG   SenderThread:72042 [sender.py:send():379] send: files
+2024-12-17 21:03:34,382 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_a2d5dd163e6dc31055f4.mp4 with policy now
+2024-12-17 21:03:34,383 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:03:34,384 DEBUG   SenderThread:72042 [sender.py:send():379] send: history
+2024-12-17 21:03:34,384 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:03:34,384 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:03:34,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:03:34,445 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:03:34,465 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:34,640 INFO    wandb-upload_1:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/x4twrsoe-media/videos/eval/1_eval_1_a2d5dd163e6dc31055f4.mp4
+2024-12-17 21:03:35,067 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json
+2024-12-17 21:03:35,067 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:35,067 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/1_eval_1_a2d5dd163e6dc31055f4.mp4
+2024-12-17 21:03:35,068 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval
+2024-12-17 21:03:35,354 INFO    wandb-upload_0:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/5pi7yt8w-media/videos/eval/1_eval_1_a2d5dd163e6dc31055f4.mp4
+2024-12-17 21:03:35,463 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:35,571 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:03:36,467 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:37,068 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:37,460 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:38,466 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:39,068 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:39,472 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:40,461 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:41,321 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:03:41,462 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:42,462 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:43,070 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:43,462 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:43,824 DEBUG   SenderThread:72042 [sender.py:send():379] send: history
+2024-12-17 21:03:43,824 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:03:43,824 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:03:44,070 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json
+2024-12-17 21:03:44,070 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0
+2024-12-17 21:03:44,333 INFO    wandb-upload_1:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/2e8vua53-events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0
+2024-12-17 21:03:44,463 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:45,071 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:45,462 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:46,462 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:46,597 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:03:47,071 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:47,462 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:48,463 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:49,072 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:49,440 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:03:49,441 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:03:49,485 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:50,463 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:51,464 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:52,464 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:52,550 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:03:53,464 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:54,464 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:55,464 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:56,464 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:57,464 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:57,767 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:03:58,464 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:59,075 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:59,464 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:00,464 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:01,464 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:02,465 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:03,446 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:04:03,465 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:04,110 DEBUG   SenderThread:72042 [sender.py:send():379] send: stats
+2024-12-17 21:04:04,440 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:04:04,441 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:04:04,485 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:05,465 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:06,465 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:07,465 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:08,465 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:08,589 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:04:09,466 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:10,466 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:11,466 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:12,466 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:13,466 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:14,174 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:04:14,466 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:15,080 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:04:15,473 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:15,565 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:04:15,566 DEBUG   SenderThread:72042 [sender.py:send():379] send: files
+2024-12-17 21:04:15,566 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_c3a9936bb29a61a5d383.mp4 with policy now
+2024-12-17 21:04:15,566 DEBUG   SenderThread:72042 [sender.py:send():379] send: history
+2024-12-17 21:04:15,566 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:04:15,567 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:04:15,821 INFO    wandb-upload_0:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/t3dddxba-media/videos/eval/2_eval_2_c3a9936bb29a61a5d383.mp4
+2024-12-17 21:04:16,080 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json
+2024-12-17 21:04:16,081 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/2_eval_2_c3a9936bb29a61a5d383.mp4
+2024-12-17 21:04:16,081 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval
+2024-12-17 21:04:16,348 INFO    wandb-upload_1:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/7ioqhnbm-media/videos/eval/2_eval_2_c3a9936bb29a61a5d383.mp4
+2024-12-17 21:04:16,473 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:17,081 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:04:17,476 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:18,081 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:04:18,477 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:19,447 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:04:19,447 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:04:19,452 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:04:19,489 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:20,476 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:21,082 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:04:21,501 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:22,482 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:23,083 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:04:23,477 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:24,447 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:04:24,476 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:25,478 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:26,477 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:26,659 DEBUG   SenderThread:72042 [sender.py:send():379] send: history
+2024-12-17 21:04:26,659 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:04:26,661 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:04:27,084 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json
+2024-12-17 21:04:27,084 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0
+2024-12-17 21:04:27,477 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:28,479 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:29,085 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:04:29,449 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:04:29,479 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:30,478 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:31,085 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:04:31,478 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:32,478 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:33,086 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:04:33,478 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:34,110 DEBUG   SenderThread:72042 [sender.py:send():379] send: stats
+2024-12-17 21:04:34,441 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:04:34,441 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:04:34,485 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:34,566 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:04:35,478 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:36,478 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:37,478 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:38,479 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:39,479 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:40,447 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:04:40,479 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:41,479 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:42,479 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:43,479 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:44,479 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:45,447 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:04:45,479 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:46,480 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:47,480 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:48,480 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:49,441 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:04:49,441 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:04:49,485 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:50,482 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:50,516 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:04:51,480 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:52,480 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:53,481 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:54,481 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:55,481 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:56,448 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:04:56,481 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:57,482 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:58,254 DEBUG   SenderThread:72042 [sender.py:send():379] send: files
+2024-12-17 21:04:58,254 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_a5ebabbd104fafab5ecd.mp4 with policy now
+2024-12-17 21:04:58,257 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:04:58,257 DEBUG   SenderThread:72042 [sender.py:send():379] send: history
+2024-12-17 21:04:58,257 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:04:58,258 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:04:58,487 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:58,503 INFO    wandb-upload_0:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/0p31b03r-media/videos/eval/3_eval_3_a5ebabbd104fafab5ecd.mp4
+2024-12-17 21:04:59,094 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json
+2024-12-17 21:04:59,095 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:04:59,095 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/3_eval_3_a5ebabbd104fafab5ecd.mp4
+2024-12-17 21:04:59,095 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval
+2024-12-17 21:04:59,382 INFO    wandb-upload_1:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/1nan1h56-media/videos/eval/3_eval_3_a5ebabbd104fafab5ecd.mp4
+2024-12-17 21:04:59,485 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:00,485 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:01,095 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:01,487 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:02,445 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:02,500 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:03,096 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:03,491 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:04,111 DEBUG   SenderThread:72042 [sender.py:send():379] send: stats
+2024-12-17 21:05:04,446 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:05:04,446 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:05:04,492 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:05,096 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:05,490 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:06,493 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:06,707 DEBUG   SenderThread:72042 [sender.py:send():379] send: history
+2024-12-17 21:05:06,707 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:05:06,708 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:05:07,097 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json
+2024-12-17 21:05:07,097 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:07,097 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0
+2024-12-17 21:05:07,355 INFO    wandb-upload_0:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/fk5308hp-events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0
+2024-12-17 21:05:07,497 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:07,652 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:08,488 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:09,098 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:09,498 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:10,488 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:11,488 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:12,488 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:12,890 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:13,099 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:13,488 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:14,488 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:15,488 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:16,488 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:17,488 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:18,449 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:18,488 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:19,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:05:19,443 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:05:19,489 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:20,488 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:21,489 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:22,489 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:23,489 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:23,633 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:24,489 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:25,103 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:25,489 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:26,489 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:27,104 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:27,489 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:28,489 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:28,633 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:29,489 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:30,490 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:31,490 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:32,490 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:33,106 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:33,490 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:34,111 DEBUG   SenderThread:72042 [sender.py:send():379] send: stats
+2024-12-17 21:05:34,111 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:34,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:05:34,443 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:05:34,490 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:35,490 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:36,490 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:37,490 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:38,493 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:38,512 DEBUG   SenderThread:72042 [sender.py:send():379] send: files
+2024-12-17 21:05:38,512 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_a10c49f08e3e38202133.mp4 with policy now
+2024-12-17 21:05:38,553 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:05:38,553 DEBUG   SenderThread:72042 [sender.py:send():379] send: history
+2024-12-17 21:05:38,554 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:05:38,554 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:05:38,775 INFO    wandb-upload_1:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/4al7tj4q-media/videos/eval/4_eval_4_a10c49f08e3e38202133.mp4
+2024-12-17 21:05:39,108 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json
+2024-12-17 21:05:39,108 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:39,108 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/4_eval_4_a10c49f08e3e38202133.mp4
+2024-12-17 21:05:39,108 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval
+2024-12-17 21:05:39,374 INFO    wandb-upload_0:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/tmxkx7jf-media/videos/eval/4_eval_4_a10c49f08e3e38202133.mp4
+2024-12-17 21:05:39,453 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:39,520 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:40,503 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:41,109 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:41,508 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:42,510 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:43,109 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:43,511 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:44,472 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:44,508 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:45,508 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:46,513 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:47,111 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:47,512 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:48,522 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:49,033 DEBUG   SenderThread:72042 [sender.py:send():379] send: exit
+2024-12-17 21:05:49,033 INFO    SenderThread:72042 [sender.py:send_exit():586] handling exit code: 0
+2024-12-17 21:05:49,034 INFO    SenderThread:72042 [sender.py:send_exit():588] handling runtime: 224
+2024-12-17 21:05:49,034 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:05:49,034 INFO    SenderThread:72042 [sender.py:send_exit():594] send defer
+2024-12-17 21:05:49,034 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:49,034 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-17 21:05:49,034 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:49,034 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-17 21:05:49,034 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 1
+2024-12-17 21:05:49,034 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:49,034 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-17 21:05:49,035 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:49,035 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-17 21:05:49,035 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 2
+2024-12-17 21:05:49,035 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:49,035 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-17 21:05:49,035 INFO    HandlerThread:72042 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-17 21:05:49,035 DEBUG   SystemMonitor:72042 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-17 21:05:49,035 INFO    HandlerThread:72042 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-17 21:05:49,035 DEBUG   SystemMonitor:72042 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-17 21:05:49,035 INFO    HandlerThread:72042 [interfaces.py:finish():200] Joined disk monitor
+2024-12-17 21:05:49,081 INFO    HandlerThread:72042 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-17 21:05:49,082 INFO    HandlerThread:72042 [interfaces.py:finish():200] Joined memory monitor
+2024-12-17 21:05:49,082 INFO    HandlerThread:72042 [interfaces.py:finish():200] Joined network monitor
+2024-12-17 21:05:49,082 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:49,082 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-17 21:05:49,082 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 3
+2024-12-17 21:05:49,082 DEBUG   SenderThread:72042 [sender.py:send():379] send: stats
+2024-12-17 21:05:49,083 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:49,083 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-17 21:05:49,083 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:49,083 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-17 21:05:49,083 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 4
+2024-12-17 21:05:49,083 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:49,083 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-17 21:05:49,112 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json
+2024-12-17 21:05:49,112 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0
+2024-12-17 21:05:51,112 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:55,085 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:05:55,085 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:55,085 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:55,085 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-17 21:05:55,085 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-17 21:05:55,085 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:55,085 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 5
+2024-12-17 21:05:55,086 DEBUG   SenderThread:72042 [sender.py:send():379] send: history
+2024-12-17 21:05:55,086 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:55,086 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:05:55,087 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-17 21:05:55,087 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:05:55,087 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:05:55,088 DEBUG   SenderThread:72042 [sender.py:send():379] send: history
+2024-12-17 21:05:55,088 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:05:55,091 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:05:55,091 DEBUG   SenderThread:72042 [sender.py:send():379] send: summary
+2024-12-17 21:05:55,091 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:05:55,092 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:55,092 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-17 21:05:55,092 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 6
+2024-12-17 21:05:55,092 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:55,092 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-17 21:05:55,092 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:55,092 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-17 21:05:55,092 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 7
+2024-12-17 21:05:55,092 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:55,092 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:55,092 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-17 21:05:55,092 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:55,092 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-17 21:05:55,114 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json
+2024-12-17 21:05:56,035 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:05:57,659 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 8
+2024-12-17 21:05:57,659 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:05:57,659 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:57,660 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-17 21:05:57,660 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:57,660 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-17 21:05:57,660 INFO    SenderThread:72042 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-17 21:05:57,662 INFO    SenderThread:72042 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-17 21:05:57,676 INFO    SenderThread:72042 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-17 21:05:57,690 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 9
+2024-12-17 21:05:57,690 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:57,690 DEBUG   SenderThread:72042 [sender.py:send():379] send: artifact
+2024-12-17 21:05:57,690 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-17 21:05:58,035 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:05:58,115 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:58,247 INFO    wandb-upload_2:72042 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmtj1ivhh
+2024-12-17 21:05:58,247 INFO    wandb-upload_0:72042 [upload_job.py:push():85] Skipped uploading /tmp/tmpq89dbjgm/wandb-job.json
+2024-12-17 21:05:58,372 INFO    wandb-upload_1:72042 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6fe2von7
+2024-12-17 21:05:58,978 INFO    SenderThread:72042 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM4MzExNzUzMw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM4MzA4NzA1NA==', 'versionIndex': 375}}}
+2024-12-17 21:05:58,978 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:58,978 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-17 21:05:58,978 INFO    SenderThread:72042 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-17 21:05:59,115 INFO    SenderThread:72042 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files
+2024-12-17 21:05:59,115 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/config.yaml config.yaml
+2024-12-17 21:05:59,115 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0 events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0
+2024-12-17 21:05:59,117 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json wandb-summary.json
+2024-12-17 21:05:59,117 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/requirements.txt requirements.txt
+2024-12-17 21:05:59,117 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-metadata.json wandb-metadata.json
+2024-12-17 21:05:59,117 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log output.log
+2024-12-17 21:05:59,117 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/diff.patch diff.patch
+2024-12-17 21:05:59,118 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-17 21:05:59,122 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/2_eval_2_c3a9936bb29a61a5d383.mp4 media/videos/eval/2_eval_2_c3a9936bb29a61a5d383.mp4
+2024-12-17 21:05:59,123 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/0_eval_0_c14cbfe41183ade1bc4f.mp4 media/videos/eval/0_eval_0_c14cbfe41183ade1bc4f.mp4
+2024-12-17 21:05:59,123 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/4_eval_4_a10c49f08e3e38202133.mp4 media/videos/eval/4_eval_4_a10c49f08e3e38202133.mp4
+2024-12-17 21:05:59,123 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/3_eval_3_a5ebabbd104fafab5ecd.mp4 media/videos/eval/3_eval_3_a5ebabbd104fafab5ecd.mp4
+2024-12-17 21:05:59,124 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/1_eval_1_a2d5dd163e6dc31055f4.mp4 media/videos/eval/1_eval_1_a2d5dd163e6dc31055f4.mp4
+2024-12-17 21:05:59,124 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 10
+2024-12-17 21:05:59,124 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:05:59,124 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:59,124 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-17 21:05:59,124 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:59,124 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-17 21:05:59,124 INFO    SenderThread:72042 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 21:05:59,299 INFO    wandb-upload_0:72042 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json
+2024-12-17 21:05:59,314 INFO    wandb-upload_2:72042 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/config.yaml
+2024-12-17 21:05:59,352 INFO    wandb-upload_1:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/cd9a6ukz-events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0
+2024-12-17 21:05:59,353 INFO    wandb-upload_4:72042 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:59,391 INFO    wandb-upload_3:72042 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/requirements.txt
+2024-12-17 21:05:59,591 INFO    Thread-11 (_thread_body):72042 [sender.py:transition_state():614] send defer: 11
+2024-12-17 21:05:59,591 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:59,591 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-17 21:05:59,592 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:59,592 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-17 21:05:59,592 INFO    SenderThread:72042 [file_pusher.py:join():175] waiting for file pusher
+2024-12-17 21:05:59,593 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 12
+2024-12-17 21:05:59,593 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:59,593 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-17 21:05:59,593 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:59,593 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-17 21:05:59,593 INFO    SenderThread:72042 [file_stream.py:finish():601] file stream finish called
+2024-12-17 21:05:59,911 INFO    SenderThread:72042 [file_stream.py:finish():605] file stream finish is done
+2024-12-17 21:05:59,911 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 13
+2024-12-17 21:05:59,911 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:59,912 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-17 21:05:59,912 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:59,912 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-17 21:05:59,912 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 14
+2024-12-17 21:05:59,912 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:59,912 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-17 21:05:59,912 DEBUG   SenderThread:72042 [sender.py:send():379] send: final
+2024-12-17 21:05:59,912 DEBUG   SenderThread:72042 [sender.py:send():379] send: footer
+2024-12-17 21:05:59,912 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:59,912 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-17 21:05:59,913 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:05:59,913 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:05:59,913 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: server_info
+2024-12-17 21:05:59,913 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: server_info
+2024-12-17 21:05:59,915 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:05:59,915 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-17 21:05:59,915 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-17 21:05:59,915 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:59,986 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:05:59,986 INFO    MainThread:72042 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-17 21:05:59,986 INFO    MainThread:72042 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-17 21:05:59,986 INFO    MainThread:72042 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-17 21:05:59,987 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-17 21:05:59,987 INFO    HandlerThread:72042 [handler.py:finish():882] shutting down handler
+2024-12-17 21:06:00,915 INFO    WriterThread:72042 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/run-tbm1o8qf.wandb
+2024-12-17 21:06:00,986 INFO    SenderThread:72042 [sender.py:finish():1615] shutting down sender
+2024-12-17 21:06:00,986 INFO    SenderThread:72042 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 21:06:00,986 INFO    SenderThread:72042 [file_pusher.py:join():175] waiting for file pusher

210157/wandb/debug.log

@@ -0,0 +1,31 @@
+2024-12-17 21:02:03,774 INFO    MainThread:71857 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-17 21:02:03,774 INFO    MainThread:71857 [wandb_setup.py:_flush():76] Configure stats pid to 71857
+2024-12-17 21:02:03,774 INFO    MainThread:71857 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-17 21:02:03,774 INFO    MainThread:71857 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/settings
+2024-12-17 21:02:03,774 INFO    MainThread:71857 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-17 21:02:03,774 INFO    MainThread:71857 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-17 21:02:03,774 INFO    MainThread:71857 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-17 21:02:03,774 INFO    MainThread:71857 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-17 21:02:03,774 INFO    MainThread:71857 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/logs/debug.log
+2024-12-17 21:02:03,774 INFO    MainThread:71857 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/logs/debug-internal.log
+2024-12-17 21:02:03,775 INFO    MainThread:71857 [wandb_init.py:init():569] calling init triggers
+2024-12-17 21:02:03,775 INFO    MainThread:71857 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '507_0', 'wandb_notes': '507_0_restarted_1', 'eval': True, 'true_action_history': False, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': False, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['greece_twodim'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/507_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157'}
+2024-12-17 21:02:03,775 INFO    MainThread:71857 [wandb_init.py:init():619] starting backend
+2024-12-17 21:02:03,775 INFO    MainThread:71857 [wandb_init.py:init():623] setting up manager
+2024-12-17 21:02:03,779 INFO    MainThread:71857 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-17 21:02:03,780 INFO    MainThread:71857 [wandb_init.py:init():631] backend started and connected
+2024-12-17 21:02:03,792 INFO    MainThread:71857 [wandb_init.py:init():720] updated telemetry
+2024-12-17 21:02:03,799 INFO    MainThread:71857 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-17 21:02:04,055 INFO    MainThread:71857 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-17 21:02:04,096 INFO    MainThread:71857 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-17 21:02:04,096 INFO    MainThread:71857 [wandb_init.py:init():804] starting run threads in backend
+2024-12-17 21:02:04,439 INFO    MainThread:71857 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-17 21:02:04,440 INFO    MainThread:71857 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-17 21:02:04,440 INFO    MainThread:71857 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-17 21:02:04,440 INFO    MainThread:71857 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-17 21:02:04,443 INFO    MainThread:71857 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-17 21:02:04,444 INFO    MainThread:71857 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/tb, True
+2024-12-17 21:02:11,983 INFO    MainThread:71857 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'}
+2024-12-17 21:06:00,988 WARNING MsgRouterThr:71857 [router.py:message_loop():77] message_loop has been closed

210157/wandb/run-20241217_210203-tbm1o8qf/files/code/FISH/eval_robot.py

@@ -0,0 +1,512 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)		
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()

210157/wandb/run-20241217_210203-tbm1o8qf/files/config.yaml

@@ -0,0 +1,891 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 0
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '507_0'
+wandb_notes:
+  desc: null
+  value: 507_0_restarted_1
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: false
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: false
+use_sdf_maps:
+  desc: null
+  value: false
+use_normals_maps:
+  desc: null
+  value: false
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value:
+  - greece_twodim
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 0
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/kostas-graid/datasets/extrinsic_contact_data/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/507_0
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: false
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: false
+        use_sdf_maps: false
+        use_normals_maps: false
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 0
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: false
+          use_sdf_maps: false
+          use_normals_maps: false
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1734487323
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 16
+      - 23
+      - 35
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: greece
+left_book_slot:
+  desc: null
+  value: twodim

210157/wandb/run-20241217_210203-tbm1o8qf/files/diff.patch

@@ -0,0 +1,195 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index bb0a413..abfa412 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -107,7 +107,7 @@ num_valid_demos: null
+ val_num_groups: 3
+ 
+ # name_of_expert_demo: 'book_insertion_wo_ends_wedge_v2_76_all_demos_10hz_no_filter_wo_flow_mask_expert_demos_imp_act'
+-name_of_expert_demo: '192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act'
++name_of_expert_demo: '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act'
+ expert_dataset_dirpath: '${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}'
+ 
+ expert_dataset: '${expert_dataset_dirpath}/demos.zarr'
+@@ -123,89 +123,28 @@ bc_weight_type: 'qfilter' # linear, qfilter
+ 
+ # Load weights
+ load_checkpoint: ${agent.load_checkpoint}
+-# 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# full 32 demos
+-# wandb_run_id: "265625" # RGBD history seed 0
+-# wandb_run_id: "266370_1" # RGBD history seed 1
+-# wandb_run_id: "266370_2" # RGBD history seed 2
+-# wandb_run_id: "266382_4" # RGBD history seed 4
+-
+-# 16 held out demos (16 demos)
+-# wandb_run_id: "266392_0" # RGBD+act history dataset seed 0
+-# wandb_run_id: "266407_0" # RGBD+act history dataset seed 1
+-wandb_run_id: "266406_0" # RGBD+act history dataset seed 2
+-
+-# wandb_run_id: "265626" # RGBD+Mask+act history seed 0
+-# wandb_run_id: "265638" # RGBD+Mask+act history seed 1
+-# wandb_run_id: "265637" # RGBD+Mask+act history seed 2
+-# wandb_run_id: "265636" # RGBD+Mask+act history seed 3
+-# wandb_run_id: "265635" # RGBD+Mask+act history seed 4
+-
+-# wandb_run_id: "265628" # RGBD+Mask+contact+act history seed 0
+-# wandb_run_id: "265630" # RGBD+Mask+contact+act history seed 1
+-# wandb_run_id: "265631" # RGBD+Mask+contact+act history seed 2
+-# wandb_run_id: "265633" # RGBD+Mask+contact+act history seed 3
+-# wandb_run_id: "265634" # RGBD+Mask+contact+act history seed 4
+-
+-# 8 held out demos (26 demos)
+-# wandb_run_id: "265652" # RGBD+Mask+contact+act history dataset seed 0
+-# wandb_run_id: "265654" # RGBD+Mask+contact+act history dataset seed 1
+-# wandb_run_id: "265655" # RGBD+Mask+contact+act history dataset seed 2
+-
+-# 8 held out demos (26 demos)
+-# wandb_run_id: "265656" # RGBD+Mask+act history dataset seed 0
+-# wandb_run_id: "265657" # RGBD+Mask+act history dataset seed 1
+-# wandb_run_id: "265658" # RGBD+Mask+act history dataset seed 2
+-
+-# 16 held out demos (16 demos)
+-# wandb_run_id: "265641" # RGBD+Mask+act history dataset seed 0
+-# wandb_run_id: "265644" # RGBD+Mask+act history dataset seed 1
+-# wandb_run_id: "265645" # RGBD+Mask+act history dataset seed 2
+-# wandb_run_id: "266381_0" # RGBD+Mask+act history dataset seed 4
+-
+-# 16 held out demos (16 demos)
+-# wandb_run_id: "265646" # RGBD+Mask+Contact+act history dataset seed 0
+-# wandb_run_id: "265649" # RGBD+Mask+Contact+act history dataset seed 1
+-# wandb_run_id: "265650" # RGBD+Mask+Contact+act history dataset seed 2
+-# wandb_run_id: "266378_0" # RGBD+Mask+Contact+act history dataset seed 4
+-
+-
+-
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# full dataset
+-# 6 step action history
+-# wandb_run_id: '250358_0' # RGBD+act history
+-# wandb_run_id: '250371_0' # RGBD+mask+act history
+-# wandb_run_id: '250366_0' # RGBD+contact+act history
+-# wandb_run_id: '250376_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250366_0' # RGBD+contact+act history
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# 16 held out demos
+-# wandb_run_id: '250362_0' # RGBD+act history
+-# wandb_run_id: '250373_0' # RGBD+mask+act history
+-# wandb_run_id: '250378_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250368_0' # RGBD+contact+act history
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# 32 held out demos
+-# wandb_run_id: '250365_0' # RGBD+act history
+-# wandb_run_id: '250375_0' # RGBD+mask+act history
+-wandb_run_id: '250380_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250370_0' # RGBD+contact+act history
+-# 4 step action history
+-# wandb_run_id: '264772_0' # RGBD+act history
+-# wandb_run_id: '264775_0' # RGBD+mask+act history
+-# wandb_run_id: '264776_0' # RGBD+contact+act history
+-# wandb_run_id: '264777_0' # RGBD+mask+contact+act history
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# 16 held out demos
+-# wandb_run_id: '250362_0' # RGBD+act history
+-# wandb_run_id: '250373_0' # RGBD+mask+act history
+-# wandb_run_id: '250378_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250368_0' # RGBD+contact+act history
++# 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
++# 14 demos
++
++# RGBD+act history 
++# wandb_run_id: '497_0' # german
++# wandb_run_id: '505_0' # hbm
++# wandb_run_id: '506_0' # lady
++wandb_run_id: '507_0' # greece
++# wandb_run_id: '508_0' # fowlers
++# wandb_run_id: '509_0' # lib
++# wandb_run_id: '510_0' # modelsys
++# wandb_run_id: '511_0' # electrodyn
++
++# RGBD+mask+act history
++# wandb_run_id: '512_0' # german
++# wandb_run_id: '520_0' # hbm
++# wandb_run_id: '521_0' # lady
++# wandb_run_id: '522_0' # greece
++# wandb_run_id: '523_0' # fowlers
++# wandb_run_id: '524_0' # lib
++# wandb_run_id: '525_0' # modelsys
++# wandb_run_id: '526_0' # electrodyn
+ 
+ # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # 32 held out demos
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index 9406a46..0f225ec 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -15,12 +15,24 @@ if not local_checkpoint_root_dir.exists():
+     local_checkpoint_root_dir.mkdir(parents=True)
+ 
+ # Download the model checkpoints
+-remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1')
++# remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1')
++remote_checkpoint_root_dir = Path('/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1')
+ remote_checkpoint_root_dir = remote_checkpoint_root_dir.expanduser()
+ remote_username = 'leonmkim'
+-remote_host = 'halley'
++remote_host = 'grasp-login1'
+ 
+ #180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
++# run_id_list = [
++#     '512_0', # german
++#     '520_0', # hbm
++#     '521_0', # lady
++#     '522_0', # greece
++#     '523_0', # fowlers
++#     '524_0', # lib
++#     '525_0', # modelsys
++#     '526_0', # electrodyn
++# ]
++
+ # run_id_list = [
+ #     '265638',
+ #     '265637',
+@@ -75,6 +87,17 @@ remote_host = 'halley'
+ # ]
+ 
+ # 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
++# run_id_list = [
++#     '497_0', # german
++#     '505_0', # hbm
++#     '506_0', # lady
++#     '507_0', # greece
++#     '508_0', # fowlers
++#     '509_0', # lib
++#     '510_0', # modelsys
++#     '511_0', # electrodyn
++# ]
++
+ # run_id_list = [
+ #     '266382_4',
+ #     '266370_2',
+@@ -116,13 +139,13 @@ remote_host = 'halley'
+ # ]
+ 
+ # 180x240_1_RGB_D_2.0_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+-run_id_list = [
+-    # '266384_4,# not yet available
+-    '266372_1',
+-    '266372_2',
+-    '265627',
+-    '265620',
+-]
++# run_id_list = [
++#     # '266384_4,# not yet available
++#     '266372_1',
++#     '266372_2',
++#     '265627',
++#     '265620',
++# ]
+ checkpoint_epoch = 12000
+ 
+ # use subprocess to download the model checkpoints in parallel

210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/0_eval_0_c14cbfe41183ade1bc4f.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c14cbfe41183ade1bc4ff61f35119dcc66addcd0c91751b9973014f71084eb34
+size 1137780

210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/1_eval_1_a2d5dd163e6dc31055f4.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2d5dd163e6dc31055f490b0a07dc67d943975e1db66596df8c9907747fff5dc
+size 1132118

210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/2_eval_2_c3a9936bb29a61a5d383.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3a9936bb29a61a5d383a2ded237402015093b53dcab382212c28520b1b10874
+size 1135489

210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/3_eval_3_a5ebabbd104fafab5ecd.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5ebabbd104fafab5ecd21d62ee8f1c213976ba38e174b9c18f59177a29e256b
+size 1145514

210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/4_eval_4_a10c49f08e3e38202133.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a10c49f08e3e38202133932429d20d6b9990c27d3c533d8f6dade92cf1d8175c
+size 1133992

210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log

@@ -0,0 +1,220 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1734487331.984495]: resetting environment
+[INFO] [1734487331.985460]: cleared current plan
+[INFO] [1734487331.986587]: moving to home
+[INFO] [1734487332.989281]: reached home
+[INFO] [1734487332.990405]: reset action history
+[INFO] [1734487335.697899]: environment reset
+Starting episode 0
+[INFO] [1734487335.699895]: resetting environment
+[INFO] [1734487335.700758]: cleared current plan
+[INFO] [1734487335.701448]: moving to home
+[INFO] [1734487336.704172]: reached home
+[INFO] [1734487336.707172]: reset action history
+[INFO] [1734487339.415215]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode![INFO] [1734487343.935635]: resetting environment
+[INFO] [1734487343.940694]: cleared current plan
+[INFO] [1734487343.941671]: moving to home
+[INFO] [1734487344.944332]: reached home
+[INFO] [1734487344.946218]: reset action history
+[INFO] [1734487347.652715]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/episode_rosbags/episode_0_2024-12-17-21-02-27.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-71857.log
+started roslaunch server http://158.130.50.37:42193/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [72162]
+process[print_text-2]: started with pid [72163]
+process[rosbag_record-3]: started with pid [72164]
+started bagging!
+For topic gripper_width: timestamp difference is -103273800 for nearest: 1734487351919746706 - target: 1734487352023020506 at idx 52
+[INFO] [1734487374.460503]: Storing episode...
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit[pub_text-1] killing on exit
+[INFO] [1734487375.199737]: Stored episode 1.
+[INFO] [1734487375.199981]: Saving video...
+[INFO] [1734487375.395184]: Video saved!
+Starting episode 1
+[INFO] [1734487375.406316]: resetting environment
+[INFO] [1734487375.406581]: cleared current plan
+[INFO] [1734487375.407089]: moving to home
+[INFO] [1734487378.409795]: reached home
+[INFO] [1734487378.410609]: reset action history
+[INFO] [1734487381.117858]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1734487382.607314]: resetting environment
+[INFO] [1734487382.608975]: cleared current plan
+[INFO] [1734487382.610228]: moving to home
+[INFO] [1734487383.612283]: reached home
+[INFO] [1734487383.613288]: reset action history
+[INFO] [1734487386.320426]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/episode_rosbags/episode_1_2024-12-17-21-03-06.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-71857.log
+started roslaunch server http://158.130.50.37:42109/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [72268]
+process[print_text-5]: started with pid [72269]
+process[rosbag_record-6]: started with pid [72270]
+started bagging!
+[INFO] [1734487413.057734]: Storing episode...
+[rosbag_record-6] killing on exit
+[pub_text-4] killing on exit
+[print_text-5] killing on exit
+[INFO] [1734487413.840821]: Stored episode 2.
+[INFO] [1734487413.843784]: Saving video...
+[INFO] [1734487414.357016]: Video saved!
+Starting episode 2
+[INFO] [1734487414.387126]: resetting environment
+[INFO] [1734487414.387410]: cleared current plan
+[INFO] [1734487414.393545]: moving to home
+[INFO] [1734487417.507396]: reached home
+[INFO] [1734487417.512089]: reset action history
+[INFO] [1734487420.280764]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1734487423.550663]: resetting environment
+[INFO] [1734487423.550908]: cleared current plan
+[INFO] [1734487423.553848]: moving to home
+[INFO] [1734487424.556807]: reached home
+[INFO] [1734487424.557585]: reset action history
+[INFO] [1734487427.264418]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/episode_rosbags/episode_2_2024-12-17-21-03-47.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-71857.log
+started roslaunch server http://158.130.50.37:41123/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [72422]
+process[print_text-8]: started with pid [72423]
+process[rosbag_record-9]: started with pid [72424]
+started bagging!
+For topic gripper_width: timestamp difference is -102760032 for nearest: 1734487436819787070 - target: 1734487436922547102 at idx 124
+For topic gripper_width: timestamp difference is -101120183 for nearest: 1734487437619787982 - target: 1734487437720908165 at idx 133
+For topic gripper_width: timestamp difference is -100731448 for nearest: 1734487437819787427 - target: 1734487437920518875 at idx 135
+[INFO] [1734487454.174260]: Storing episode...
+[rosbag_record-9] killing on exit[print_text-8] killing on exit
+[pub_text-7] killing on exit
+[INFO] [1734487454.950664]: Stored episode 3.
+[INFO] [1734487454.956289]: Saving video...
+[INFO] [1734487455.519263]: Video saved!
+Starting episode 3
+[INFO] [1734487455.551877]: resetting environment
+[INFO] [1734487455.556146]: cleared current plan
+[INFO] [1734487455.558032]: moving to home
+[WARN] [1734487455.521957]: Plan exhausted
+[INFO] [1734487458.666531]: reached home
+[INFO] [1734487458.670864]: reset action history
+[INFO] [1734487461.452351]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode![INFO] [1734487466.238348]: resetting environment
+[INFO] [1734487466.243072]: cleared current plan
+[INFO] [1734487466.245153]: moving to home
+[INFO] [1734487467.246952]: reached home
+[INFO] [1734487467.249122]: reset action history
+[INFO] [1734487470.019087]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/episode_rosbags/episode_3_2024-12-17-21-04-30.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-71857.log
+started roslaunch server http://158.130.50.37:45863/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-10]: started with pid [72551]
+process[print_text-11]: started with pid [72552]
+process[rosbag_record-12]: started with pid [72553]
+started bagging!
+[INFO] [1734487496.969785]: Storing episode...
+[rosbag_record-12] killing on exit
+[print_text-11] killing on exit[pub_text-10] killing on exit
+[INFO] [1734487497.744882]: Stored episode 4.
+[INFO] [1734487497.749886]: Saving video...
+[INFO] [1734487498.226401]: Video saved!
+Starting episode 4
+[INFO] [1734487498.267616]: resetting environment
+[INFO] [1734487498.272185]: cleared current plan
+[INFO] [1734487498.275995]: moving to home
+[INFO] [1734487501.387440]: reached home
+[INFO] [1734487501.405995]: reset action history
+[INFO] [1734487504.213460]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1734487506.638424]: resetting environment
+[INFO] [1734487506.643725]: cleared current plan
+[INFO] [1734487506.646480]: moving to home
+[INFO] [1734487507.651670]: reached home
+[INFO] [1734487507.655688]: reset action history
+[INFO] [1734487510.433914]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/episode_rosbags/episode_4_2024-12-17-21-05-10.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-71857.log
+started roslaunch server http://158.130.50.37:44777/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-13]: started with pid [72679]
+process[print_text-14]: started with pid [72680]
+process[rosbag_record-15]: started with pid [72681]
+started bagging!
+For topic gripper_width: timestamp difference is -121406673 for nearest: 1734487522419785143 - target: 1734487522541191816 at idx 165
+For topic gripper_width: timestamp difference is -118971270 for nearest: 1734487525419809658 - target: 1734487525538780928 at idx 200
+For topic gripper_width: timestamp difference is -114432285 for nearest: 1734487531019745877 - target: 1734487531134178162 at idx 249
+For topic gripper_width: timestamp difference is -114226504 for nearest: 1734487531219787912 - target: 1734487531334014416 at idx 249
+[INFO] [1734487537.393131]: Storing episode...
+[rosbag_record-15] killing on exit
+[print_text-14] killing on exit
+[pub_text-13] killing on exit
+[INFO] [1734487538.185450]: Stored episode 5.
+[INFO] [1734487538.190758]: Saving video...
+[INFO] [1734487538.467896]: Video saved!
+[INFO] [1734487538.516649]: resetting environment
+[INFO] [1734487538.518431]: cleared current plan
+[INFO] [1734487538.525323]: moving to home
+[INFO] [1734487541.749911]: reached home
+[INFO] [1734487541.755373]: reset action history
+[INFO] [1734487544.614532]: environment reset
+Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!

210157/wandb/run-20241217_210203-tbm1o8qf/files/requirements.txt

@@ -0,0 +1,339 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.2
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0

210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-metadata.json

@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2024-12-18T02:02:04.118743",
+    "startedAt": "2024-12-18T02:02:03.773730",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "5f28b54373d7d41e4ebace380ff1080a8ffc4051"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 3749.995375,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4500.002,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.94,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4500.013,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 373.2729034423828
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}

210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json

@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "c14cbfe41183ade1bc4ff61f35119dcc66addcd0c91751b9973014f71084eb34", "size": 1137780, "path": "media/videos/eval/0_eval_0_c14cbfe41183ade1bc4f.mp4"}, "global_step": 660, "_timestamp": 1734487548.989449, "_runtime": 225.2081229686737, "_step": 9, "eval/1_eval": {"_type": "video-file", "sha256": "a2d5dd163e6dc31055f490b0a07dc67d943975e1db66596df8c9907747fff5dc", "size": 1132118, "path": "media/videos/eval/1_eval_1_a2d5dd163e6dc31055f4.mp4"}, "eval/num_success": 0.0, "episode": 5.0, "eval/success_rate": 0.0, "eval/2_eval": {"_type": "video-file", "sha256": "c3a9936bb29a61a5d383a2ded237402015093b53dcab382212c28520b1b10874", "size": 1135489, "path": "media/videos/eval/2_eval_2_c3a9936bb29a61a5d383.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "a5ebabbd104fafab5ecd21d62ee8f1c213976ba38e174b9c18f59177a29e256b", "size": 1145514, "path": "media/videos/eval/3_eval_3_a5ebabbd104fafab5ecd.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "a10c49f08e3e38202133932429d20d6b9990c27d3c533d8f6dade92cf1d8175c", "size": 1133992, "path": "media/videos/eval/4_eval_4_a10c49f08e3e38202133.mp4"}, "_wandb": {"runtime": 224}}

210157/wandb/run-20241217_210203-tbm1o8qf/logs/debug-internal.log

@@ -0,0 +1,654 @@
+2024-12-17 21:02:03,780 INFO    StreamThr :72042 [internal.py:wandb_internal():85] W&B internal server running at pid: 72042, started at: 2024-12-17 21:02:03.780301
+2024-12-17 21:02:03,781 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status
+2024-12-17 21:02:03,792 INFO    WriterThread:72042 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/run-tbm1o8qf.wandb
+2024-12-17 21:02:03,793 DEBUG   SenderThread:72042 [sender.py:send():379] send: header
+2024-12-17 21:02:03,801 DEBUG   SenderThread:72042 [sender.py:send():379] send: run
+2024-12-17 21:02:04,040 INFO    SenderThread:72042 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files
+2024-12-17 21:02:04,040 INFO    SenderThread:72042 [sender.py:_start_run_threads():1188] run started: tbm1o8qf with start time 1734487323.781326
+2024-12-17 21:02:04,055 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: check_version
+2024-12-17 21:02:04,055 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: check_version
+2024-12-17 21:02:04,101 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: run_start
+2024-12-17 21:02:04,105 DEBUG   HandlerThread:72042 [system_info.py:__init__():26] System info init
+2024-12-17 21:02:04,105 DEBUG   HandlerThread:72042 [system_info.py:__init__():41] System info init done
+2024-12-17 21:02:04,105 INFO    HandlerThread:72042 [system_monitor.py:start():194] Starting system monitor
+2024-12-17 21:02:04,105 INFO    SystemMonitor:72042 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-17 21:02:04,106 INFO    HandlerThread:72042 [system_monitor.py:probe():214] Collecting system info
+2024-12-17 21:02:04,106 INFO    SystemMonitor:72042 [interfaces.py:start():188] Started cpu monitoring
+2024-12-17 21:02:04,106 INFO    SystemMonitor:72042 [interfaces.py:start():188] Started disk monitoring
+2024-12-17 21:02:04,107 INFO    SystemMonitor:72042 [interfaces.py:start():188] Started gpu monitoring
+2024-12-17 21:02:04,107 INFO    SystemMonitor:72042 [interfaces.py:start():188] Started memory monitoring
+2024-12-17 21:02:04,108 INFO    SystemMonitor:72042 [interfaces.py:start():188] Started network monitoring
+2024-12-17 21:02:04,118 DEBUG   HandlerThread:72042 [system_info.py:probe():152] Probing system
+2024-12-17 21:02:04,120 DEBUG   HandlerThread:72042 [system_info.py:_probe_git():137] Probing git
+2024-12-17 21:02:04,124 DEBUG   HandlerThread:72042 [system_info.py:_probe_git():145] Probing git done
+2024-12-17 21:02:04,125 DEBUG   HandlerThread:72042 [system_info.py:probe():200] Probing system done
+2024-12-17 21:02:04,125 DEBUG   HandlerThread:72042 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-18T02:02:04.118743', 'startedAt': '2024-12-18T02:02:03.773730', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '5f28b54373d7d41e4ebace380ff1080a8ffc4051'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.995375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.002, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.94, 'min': 800.0, 'max': 4700.0}, {'current': 4500.013, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 373.2729034423828}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2024-12-17 21:02:04,125 INFO    HandlerThread:72042 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-17 21:02:04,125 INFO    HandlerThread:72042 [system_monitor.py:probe():227] Publishing system info
+2024-12-17 21:02:04,125 DEBUG   HandlerThread:72042 [system_info.py:_save_code():44] Saving code
+2024-12-17 21:02:04,126 DEBUG   HandlerThread:72042 [system_info.py:_save_code():67] Saving code done
+2024-12-17 21:02:04,126 DEBUG   HandlerThread:72042 [system_info.py:_save_patches():84] Saving git patches
+2024-12-17 21:02:04,191 DEBUG   HandlerThread:72042 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-17 21:02:04,191 INFO    HandlerThread:72042 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-17 21:02:04,194 DEBUG   SenderThread:72042 [sender.py:send():379] send: files
+2024-12-17 21:02:04,195 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-17 21:02:04,195 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-17 21:02:04,195 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-17 21:02:04,435 INFO    wandb-upload_0:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/5f5971ts-wandb-metadata.json
+2024-12-17 21:02:04,439 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-17 21:02:04,439 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: python_packages
+2024-12-17 21:02:04,440 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:02:04,440 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:02:04,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:04,444 INFO    HandlerThread:72042 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/tb"
+  save: true
+}
+control {
+  relay_id: "4e4e9038b20c4fde8df6b85c0bf5638d"
+}
+_info {
+  stream_id: "tbm1o8qf"
+}
+
+2024-12-17 21:02:04,444 DEBUG   HandlerThread:72042 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-17 21:02:04,488 INFO    wandb-upload_2:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/4v1c1pqd-diff.patch
+2024-12-17 21:02:04,499 INFO    wandb-upload_1:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/k6qe8mxm-code/FISH/eval_robot.py
+2024-12-17 21:02:04,542 DEBUG   SenderThread:72042 [sender.py:send():379] send: telemetry
+2024-12-17 21:02:04,542 DEBUG   SenderThread:72042 [sender.py:send():379] send: tbrecord
+2024-12-17 21:02:04,543 DEBUG   SenderThread:72042 [sender.py:send():379] send: files
+2024-12-17 21:02:04,543 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0 with policy live
+2024-12-17 21:02:04,725 INFO    wandb-upload_0:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/fvv96vo3-events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0
+2024-12-17 21:02:05,042 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:05,042 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/requirements.txt
+2024-12-17 21:02:05,042 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-metadata.json
+2024-12-17 21:02:05,042 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/code/FISH/eval_robot.py
+2024-12-17 21:02:05,042 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/diff.patch
+2024-12-17 21:02:05,042 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0
+2024-12-17 21:02:05,043 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/code/FISH
+2024-12-17 21:02:05,043 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/code
+2024-12-17 21:02:05,440 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:06,440 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:07,041 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:07,440 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:08,440 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:09,440 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:02:09,441 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:10,441 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:11,441 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:11,984 DEBUG   SenderThread:72042 [sender.py:send():379] send: config
+2024-12-17 21:02:12,441 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:13,043 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:13,442 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:14,441 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:02:14,442 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:15,043 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:15,442 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:16,442 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:17,044 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:17,442 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:18,442 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:19,045 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:19,461 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:02:19,461 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:19,462 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:02:19,463 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:02:20,442 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:21,045 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:21,442 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:22,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:23,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:24,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:24,944 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:02:25,046 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:25,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:26,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:27,047 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:27,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:28,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:29,047 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:29,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:30,441 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:02:30,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:31,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:32,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:33,049 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:33,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:34,440 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:02:34,440 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:02:34,481 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:35,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:35,585 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:02:36,049 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/config.yaml
+2024-12-17 21:02:36,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:37,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:38,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:39,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:40,445 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:41,441 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:02:41,445 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:42,445 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:43,445 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:44,445 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:45,445 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:46,441 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:02:46,445 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:47,445 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:48,445 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:49,440 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:02:49,440 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:02:49,481 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:50,446 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:51,447 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:51,579 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:02:52,447 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:53,447 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:54,447 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:55,404 DEBUG   SenderThread:72042 [sender.py:send():379] send: files
+2024-12-17 21:02:55,404 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c14cbfe41183ade1bc4f.mp4 with policy now
+2024-12-17 21:02:55,405 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:02:55,405 DEBUG   SenderThread:72042 [sender.py:send():379] send: history
+2024-12-17 21:02:55,406 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:02:55,407 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:02:55,448 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:55,658 INFO    wandb-upload_1:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/ft8yc6fq-media/videos/eval/0_eval_0_c14cbfe41183ade1bc4f.mp4
+2024-12-17 21:02:56,055 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json
+2024-12-17 21:02:56,055 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/0_eval_0_c14cbfe41183ade1bc4f.mp4
+2024-12-17 21:02:56,055 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos
+2024-12-17 21:02:56,055 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media
+2024-12-17 21:02:56,055 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval
+2024-12-17 21:02:56,311 INFO    wandb-upload_0:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/0udxobya-media/videos/eval/0_eval_0_c14cbfe41183ade1bc4f.mp4
+2024-12-17 21:02:56,447 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:57,055 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:02:57,408 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:02:57,448 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:58,453 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:02:59,448 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:00,448 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:01,056 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:01,448 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:02,449 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:02,607 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:03:03,057 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:03,057 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0
+2024-12-17 21:03:03,449 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:04,108 DEBUG   SystemMonitor:72042 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-17 21:03:04,109 DEBUG   SenderThread:72042 [sender.py:send():379] send: stats
+2024-12-17 21:03:04,441 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:03:04,441 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:03:04,449 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:05,058 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:05,450 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:06,450 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:07,450 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:08,450 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:08,549 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:03:09,059 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:09,450 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:10,450 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:11,450 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:12,451 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:13,451 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:14,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:03:14,452 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:15,452 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:16,452 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:17,452 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:18,452 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:19,440 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:03:19,440 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:03:19,481 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:19,578 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:03:20,452 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:21,454 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:22,455 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:23,455 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:24,455 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:25,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:03:25,455 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:26,455 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:27,456 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:28,456 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:29,456 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:30,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:03:30,457 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:31,457 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:32,457 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:33,457 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:34,109 DEBUG   SenderThread:72042 [sender.py:send():379] send: stats
+2024-12-17 21:03:34,382 DEBUG   SenderThread:72042 [sender.py:send():379] send: files
+2024-12-17 21:03:34,382 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_a2d5dd163e6dc31055f4.mp4 with policy now
+2024-12-17 21:03:34,383 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:03:34,384 DEBUG   SenderThread:72042 [sender.py:send():379] send: history
+2024-12-17 21:03:34,384 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:03:34,384 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:03:34,444 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:03:34,445 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:03:34,465 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:34,640 INFO    wandb-upload_1:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/x4twrsoe-media/videos/eval/1_eval_1_a2d5dd163e6dc31055f4.mp4
+2024-12-17 21:03:35,067 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json
+2024-12-17 21:03:35,067 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:35,067 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/1_eval_1_a2d5dd163e6dc31055f4.mp4
+2024-12-17 21:03:35,068 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval
+2024-12-17 21:03:35,354 INFO    wandb-upload_0:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/5pi7yt8w-media/videos/eval/1_eval_1_a2d5dd163e6dc31055f4.mp4
+2024-12-17 21:03:35,463 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:35,571 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:03:36,467 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:37,068 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:37,460 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:38,466 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:39,068 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:39,472 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:40,461 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:41,321 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:03:41,462 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:42,462 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:43,070 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:43,462 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:43,824 DEBUG   SenderThread:72042 [sender.py:send():379] send: history
+2024-12-17 21:03:43,824 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:03:43,824 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:03:44,070 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json
+2024-12-17 21:03:44,070 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0
+2024-12-17 21:03:44,333 INFO    wandb-upload_1:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/2e8vua53-events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0
+2024-12-17 21:03:44,463 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:45,071 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:45,462 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:46,462 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:46,597 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:03:47,071 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:47,462 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:48,463 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:49,072 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:49,440 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:03:49,441 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:03:49,485 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:50,463 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:51,464 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:52,464 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:52,550 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:03:53,464 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:54,464 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:55,464 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:56,464 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:57,464 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:57,767 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:03:58,464 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:03:59,075 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:03:59,464 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:00,464 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:01,464 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:02,465 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:03,446 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:04:03,465 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:04,110 DEBUG   SenderThread:72042 [sender.py:send():379] send: stats
+2024-12-17 21:04:04,440 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:04:04,441 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:04:04,485 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:05,465 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:06,465 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:07,465 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:08,465 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:08,589 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:04:09,466 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:10,466 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:11,466 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:12,466 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:13,466 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:14,174 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:04:14,466 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:15,080 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:04:15,473 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:15,565 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:04:15,566 DEBUG   SenderThread:72042 [sender.py:send():379] send: files
+2024-12-17 21:04:15,566 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_c3a9936bb29a61a5d383.mp4 with policy now
+2024-12-17 21:04:15,566 DEBUG   SenderThread:72042 [sender.py:send():379] send: history
+2024-12-17 21:04:15,566 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:04:15,567 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:04:15,821 INFO    wandb-upload_0:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/t3dddxba-media/videos/eval/2_eval_2_c3a9936bb29a61a5d383.mp4
+2024-12-17 21:04:16,080 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json
+2024-12-17 21:04:16,081 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/2_eval_2_c3a9936bb29a61a5d383.mp4
+2024-12-17 21:04:16,081 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval
+2024-12-17 21:04:16,348 INFO    wandb-upload_1:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/7ioqhnbm-media/videos/eval/2_eval_2_c3a9936bb29a61a5d383.mp4
+2024-12-17 21:04:16,473 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:17,081 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:04:17,476 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:18,081 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:04:18,477 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:19,447 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:04:19,447 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:04:19,452 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:04:19,489 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:20,476 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:21,082 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:04:21,501 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:22,482 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:23,083 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:04:23,477 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:24,447 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:04:24,476 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:25,478 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:26,477 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:26,659 DEBUG   SenderThread:72042 [sender.py:send():379] send: history
+2024-12-17 21:04:26,659 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:04:26,661 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:04:27,084 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json
+2024-12-17 21:04:27,084 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0
+2024-12-17 21:04:27,477 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:28,479 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:29,085 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:04:29,449 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:04:29,479 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:30,478 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:31,085 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:04:31,478 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:32,478 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:33,086 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:04:33,478 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:34,110 DEBUG   SenderThread:72042 [sender.py:send():379] send: stats
+2024-12-17 21:04:34,441 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:04:34,441 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:04:34,485 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:34,566 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:04:35,478 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:36,478 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:37,478 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:38,479 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:39,479 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:40,447 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:04:40,479 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:41,479 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:42,479 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:43,479 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:44,479 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:45,447 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:04:45,479 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:46,480 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:47,480 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:48,480 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:49,441 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:04:49,441 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:04:49,485 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:50,482 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:50,516 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:04:51,480 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:52,480 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:53,481 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:54,481 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:55,481 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:56,448 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:04:56,481 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:57,482 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:58,254 DEBUG   SenderThread:72042 [sender.py:send():379] send: files
+2024-12-17 21:04:58,254 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_a5ebabbd104fafab5ecd.mp4 with policy now
+2024-12-17 21:04:58,257 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:04:58,257 DEBUG   SenderThread:72042 [sender.py:send():379] send: history
+2024-12-17 21:04:58,257 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:04:58,258 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:04:58,487 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:04:58,503 INFO    wandb-upload_0:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/0p31b03r-media/videos/eval/3_eval_3_a5ebabbd104fafab5ecd.mp4
+2024-12-17 21:04:59,094 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json
+2024-12-17 21:04:59,095 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:04:59,095 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/3_eval_3_a5ebabbd104fafab5ecd.mp4
+2024-12-17 21:04:59,095 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval
+2024-12-17 21:04:59,382 INFO    wandb-upload_1:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/1nan1h56-media/videos/eval/3_eval_3_a5ebabbd104fafab5ecd.mp4
+2024-12-17 21:04:59,485 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:00,485 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:01,095 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:01,487 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:02,445 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:02,500 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:03,096 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:03,491 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:04,111 DEBUG   SenderThread:72042 [sender.py:send():379] send: stats
+2024-12-17 21:05:04,446 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:05:04,446 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:05:04,492 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:05,096 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:05,490 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:06,493 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:06,707 DEBUG   SenderThread:72042 [sender.py:send():379] send: history
+2024-12-17 21:05:06,707 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:05:06,708 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:05:07,097 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json
+2024-12-17 21:05:07,097 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:07,097 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0
+2024-12-17 21:05:07,355 INFO    wandb-upload_0:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/fk5308hp-events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0
+2024-12-17 21:05:07,497 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:07,652 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:08,488 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:09,098 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:09,498 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:10,488 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:11,488 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:12,488 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:12,890 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:13,099 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:13,488 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:14,488 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:15,488 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:16,488 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:17,488 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:18,449 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:18,488 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:19,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:05:19,443 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:05:19,489 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:20,488 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:21,489 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:22,489 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:23,489 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:23,633 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:24,489 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:25,103 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:25,489 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:26,489 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:27,104 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:27,489 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:28,489 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:28,633 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:29,489 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:30,490 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:31,490 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:32,490 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:33,106 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:33,490 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:34,111 DEBUG   SenderThread:72042 [sender.py:send():379] send: stats
+2024-12-17 21:05:34,111 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:34,443 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:05:34,443 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:05:34,490 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:35,490 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:36,490 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:37,490 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:38,493 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:38,512 DEBUG   SenderThread:72042 [sender.py:send():379] send: files
+2024-12-17 21:05:38,512 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_a10c49f08e3e38202133.mp4 with policy now
+2024-12-17 21:05:38,553 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:05:38,553 DEBUG   SenderThread:72042 [sender.py:send():379] send: history
+2024-12-17 21:05:38,554 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:05:38,554 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:05:38,775 INFO    wandb-upload_1:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/4al7tj4q-media/videos/eval/4_eval_4_a10c49f08e3e38202133.mp4
+2024-12-17 21:05:39,108 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json
+2024-12-17 21:05:39,108 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:39,108 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/4_eval_4_a10c49f08e3e38202133.mp4
+2024-12-17 21:05:39,108 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval
+2024-12-17 21:05:39,374 INFO    wandb-upload_0:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/tmxkx7jf-media/videos/eval/4_eval_4_a10c49f08e3e38202133.mp4
+2024-12-17 21:05:39,453 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:39,520 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:40,503 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:41,109 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:41,508 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:42,510 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:43,109 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:43,511 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:44,472 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:44,508 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:45,508 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:46,513 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:47,111 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:47,512 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:48,522 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:49,033 DEBUG   SenderThread:72042 [sender.py:send():379] send: exit
+2024-12-17 21:05:49,033 INFO    SenderThread:72042 [sender.py:send_exit():586] handling exit code: 0
+2024-12-17 21:05:49,034 INFO    SenderThread:72042 [sender.py:send_exit():588] handling runtime: 224
+2024-12-17 21:05:49,034 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:05:49,034 INFO    SenderThread:72042 [sender.py:send_exit():594] send defer
+2024-12-17 21:05:49,034 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:49,034 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-17 21:05:49,034 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:49,034 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-17 21:05:49,034 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 1
+2024-12-17 21:05:49,034 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:49,034 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-17 21:05:49,035 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:49,035 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-17 21:05:49,035 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 2
+2024-12-17 21:05:49,035 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:49,035 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-17 21:05:49,035 INFO    HandlerThread:72042 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-17 21:05:49,035 DEBUG   SystemMonitor:72042 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-17 21:05:49,035 INFO    HandlerThread:72042 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-17 21:05:49,035 DEBUG   SystemMonitor:72042 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-17 21:05:49,035 INFO    HandlerThread:72042 [interfaces.py:finish():200] Joined disk monitor
+2024-12-17 21:05:49,081 INFO    HandlerThread:72042 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-17 21:05:49,082 INFO    HandlerThread:72042 [interfaces.py:finish():200] Joined memory monitor
+2024-12-17 21:05:49,082 INFO    HandlerThread:72042 [interfaces.py:finish():200] Joined network monitor
+2024-12-17 21:05:49,082 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:49,082 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-17 21:05:49,082 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 3
+2024-12-17 21:05:49,082 DEBUG   SenderThread:72042 [sender.py:send():379] send: stats
+2024-12-17 21:05:49,083 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:49,083 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-17 21:05:49,083 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:49,083 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-17 21:05:49,083 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 4
+2024-12-17 21:05:49,083 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:49,083 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-17 21:05:49,112 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json
+2024-12-17 21:05:49,112 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0
+2024-12-17 21:05:51,112 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:55,085 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:05:55,085 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:55,085 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:55,085 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-17 21:05:55,085 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-17 21:05:55,085 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:55,085 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 5
+2024-12-17 21:05:55,086 DEBUG   SenderThread:72042 [sender.py:send():379] send: history
+2024-12-17 21:05:55,086 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:55,086 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:05:55,087 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-17 21:05:55,087 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:05:55,087 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:05:55,088 DEBUG   SenderThread:72042 [sender.py:send():379] send: history
+2024-12-17 21:05:55,088 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:05:55,091 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:05:55,091 DEBUG   SenderThread:72042 [sender.py:send():379] send: summary
+2024-12-17 21:05:55,091 INFO    SenderThread:72042 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:05:55,092 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:55,092 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-17 21:05:55,092 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 6
+2024-12-17 21:05:55,092 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:55,092 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-17 21:05:55,092 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:55,092 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-17 21:05:55,092 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 7
+2024-12-17 21:05:55,092 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:05:55,092 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:55,092 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-17 21:05:55,092 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:55,092 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-17 21:05:55,114 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json
+2024-12-17 21:05:56,035 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:05:57,659 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 8
+2024-12-17 21:05:57,659 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:05:57,659 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:57,660 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-17 21:05:57,660 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:57,660 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-17 21:05:57,660 INFO    SenderThread:72042 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-17 21:05:57,662 INFO    SenderThread:72042 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-17 21:05:57,676 INFO    SenderThread:72042 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-17 21:05:57,690 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 9
+2024-12-17 21:05:57,690 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:57,690 DEBUG   SenderThread:72042 [sender.py:send():379] send: artifact
+2024-12-17 21:05:57,690 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-17 21:05:58,035 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:05:58,115 INFO    Thread-12 :72042 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:58,247 INFO    wandb-upload_2:72042 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmtj1ivhh
+2024-12-17 21:05:58,247 INFO    wandb-upload_0:72042 [upload_job.py:push():85] Skipped uploading /tmp/tmpq89dbjgm/wandb-job.json
+2024-12-17 21:05:58,372 INFO    wandb-upload_1:72042 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6fe2von7
+2024-12-17 21:05:58,978 INFO    SenderThread:72042 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM4MzExNzUzMw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM4MzA4NzA1NA==', 'versionIndex': 375}}}
+2024-12-17 21:05:58,978 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:58,978 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-17 21:05:58,978 INFO    SenderThread:72042 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-17 21:05:59,115 INFO    SenderThread:72042 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files
+2024-12-17 21:05:59,115 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/config.yaml config.yaml
+2024-12-17 21:05:59,115 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0 events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0
+2024-12-17 21:05:59,117 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json wandb-summary.json
+2024-12-17 21:05:59,117 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/requirements.txt requirements.txt
+2024-12-17 21:05:59,117 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-metadata.json wandb-metadata.json
+2024-12-17 21:05:59,117 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log output.log
+2024-12-17 21:05:59,117 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/diff.patch diff.patch
+2024-12-17 21:05:59,118 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-17 21:05:59,122 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/2_eval_2_c3a9936bb29a61a5d383.mp4 media/videos/eval/2_eval_2_c3a9936bb29a61a5d383.mp4
+2024-12-17 21:05:59,123 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/0_eval_0_c14cbfe41183ade1bc4f.mp4 media/videos/eval/0_eval_0_c14cbfe41183ade1bc4f.mp4
+2024-12-17 21:05:59,123 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/4_eval_4_a10c49f08e3e38202133.mp4 media/videos/eval/4_eval_4_a10c49f08e3e38202133.mp4
+2024-12-17 21:05:59,123 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/3_eval_3_a5ebabbd104fafab5ecd.mp4 media/videos/eval/3_eval_3_a5ebabbd104fafab5ecd.mp4
+2024-12-17 21:05:59,124 INFO    SenderThread:72042 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/media/videos/eval/1_eval_1_a2d5dd163e6dc31055f4.mp4 media/videos/eval/1_eval_1_a2d5dd163e6dc31055f4.mp4
+2024-12-17 21:05:59,124 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 10
+2024-12-17 21:05:59,124 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:05:59,124 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:59,124 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-17 21:05:59,124 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:59,124 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-17 21:05:59,124 INFO    SenderThread:72042 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 21:05:59,299 INFO    wandb-upload_0:72042 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/wandb-summary.json
+2024-12-17 21:05:59,314 INFO    wandb-upload_2:72042 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/config.yaml
+2024-12-17 21:05:59,352 INFO    wandb-upload_1:72042 [upload_job.py:push():130] Uploaded file /tmp/tmp0dw4a13swandb/cd9a6ukz-events.out.tfevents.1734487324.leonmkim-ROG-Strix-G15CS-G15CS.71857.0
+2024-12-17 21:05:59,353 INFO    wandb-upload_4:72042 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/output.log
+2024-12-17 21:05:59,391 INFO    wandb-upload_3:72042 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/files/requirements.txt
+2024-12-17 21:05:59,591 INFO    Thread-11 (_thread_body):72042 [sender.py:transition_state():614] send defer: 11
+2024-12-17 21:05:59,591 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:59,591 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-17 21:05:59,592 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:59,592 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-17 21:05:59,592 INFO    SenderThread:72042 [file_pusher.py:join():175] waiting for file pusher
+2024-12-17 21:05:59,593 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 12
+2024-12-17 21:05:59,593 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:59,593 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-17 21:05:59,593 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:59,593 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-17 21:05:59,593 INFO    SenderThread:72042 [file_stream.py:finish():601] file stream finish called
+2024-12-17 21:05:59,911 INFO    SenderThread:72042 [file_stream.py:finish():605] file stream finish is done
+2024-12-17 21:05:59,911 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 13
+2024-12-17 21:05:59,911 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:59,912 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-17 21:05:59,912 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:59,912 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-17 21:05:59,912 INFO    SenderThread:72042 [sender.py:transition_state():614] send defer: 14
+2024-12-17 21:05:59,912 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:05:59,912 INFO    HandlerThread:72042 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-17 21:05:59,912 DEBUG   SenderThread:72042 [sender.py:send():379] send: final
+2024-12-17 21:05:59,912 DEBUG   SenderThread:72042 [sender.py:send():379] send: footer
+2024-12-17 21:05:59,912 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:05:59,912 INFO    SenderThread:72042 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-17 21:05:59,913 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:05:59,913 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:05:59,913 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: server_info
+2024-12-17 21:05:59,913 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: server_info
+2024-12-17 21:05:59,915 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:05:59,915 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-17 21:05:59,915 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-17 21:05:59,915 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:05:59,986 DEBUG   SenderThread:72042 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:05:59,986 INFO    MainThread:72042 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-17 21:05:59,986 INFO    MainThread:72042 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-17 21:05:59,986 INFO    MainThread:72042 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-17 21:05:59,987 DEBUG   HandlerThread:72042 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-17 21:05:59,987 INFO    HandlerThread:72042 [handler.py:finish():882] shutting down handler
+2024-12-17 21:06:00,915 INFO    WriterThread:72042 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/run-tbm1o8qf.wandb
+2024-12-17 21:06:00,986 INFO    SenderThread:72042 [sender.py:finish():1615] shutting down sender
+2024-12-17 21:06:00,986 INFO    SenderThread:72042 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 21:06:00,986 INFO    SenderThread:72042 [file_pusher.py:join():175] waiting for file pusher

210157/wandb/run-20241217_210203-tbm1o8qf/logs/debug.log

@@ -0,0 +1,31 @@
+2024-12-17 21:02:03,774 INFO    MainThread:71857 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-17 21:02:03,774 INFO    MainThread:71857 [wandb_setup.py:_flush():76] Configure stats pid to 71857
+2024-12-17 21:02:03,774 INFO    MainThread:71857 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-17 21:02:03,774 INFO    MainThread:71857 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/settings
+2024-12-17 21:02:03,774 INFO    MainThread:71857 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-17 21:02:03,774 INFO    MainThread:71857 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-17 21:02:03,774 INFO    MainThread:71857 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-17 21:02:03,774 INFO    MainThread:71857 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-17 21:02:03,774 INFO    MainThread:71857 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/logs/debug.log
+2024-12-17 21:02:03,774 INFO    MainThread:71857 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/wandb/run-20241217_210203-tbm1o8qf/logs/debug-internal.log
+2024-12-17 21:02:03,775 INFO    MainThread:71857 [wandb_init.py:init():569] calling init triggers
+2024-12-17 21:02:03,775 INFO    MainThread:71857 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '507_0', 'wandb_notes': '507_0_restarted_1', 'eval': True, 'true_action_history': False, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': False, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['greece_twodim'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/507_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157'}
+2024-12-17 21:02:03,775 INFO    MainThread:71857 [wandb_init.py:init():619] starting backend
+2024-12-17 21:02:03,775 INFO    MainThread:71857 [wandb_init.py:init():623] setting up manager
+2024-12-17 21:02:03,779 INFO    MainThread:71857 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-17 21:02:03,780 INFO    MainThread:71857 [wandb_init.py:init():631] backend started and connected
+2024-12-17 21:02:03,792 INFO    MainThread:71857 [wandb_init.py:init():720] updated telemetry
+2024-12-17 21:02:03,799 INFO    MainThread:71857 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-17 21:02:04,055 INFO    MainThread:71857 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-17 21:02:04,096 INFO    MainThread:71857 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-17 21:02:04,096 INFO    MainThread:71857 [wandb_init.py:init():804] starting run threads in backend
+2024-12-17 21:02:04,439 INFO    MainThread:71857 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-17 21:02:04,440 INFO    MainThread:71857 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-17 21:02:04,440 INFO    MainThread:71857 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-17 21:02:04,440 INFO    MainThread:71857 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-17 21:02:04,443 INFO    MainThread:71857 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-17 21:02:04,444 INFO    MainThread:71857 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/507_0/210157/tb, True
+2024-12-17 21:02:11,983 INFO    MainThread:71857 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'}
+2024-12-17 21:06:00,988 WARNING MsgRouterThr:71857 [router.py:message_loop():77] message_loop has been closed

config.yaml

@@ -0,0 +1,546 @@
+root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data
+replay_buffer_size: 150000
+replay_buffer_num_workers: 2
+nstep: 3
+batch_size: 128
+seed: 0
+dataset_shuffle_seed: 0
+device: cuda
+save_video: true
+save_train_video: true
+use_tb: true
+use_wandb: true
+wandb_run_id: '507_0'
+wandb_notes: 507_0_restarted_1
+eval: false
+true_action_history: false
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.2
+env_dtc_max_value: 0.4
+grasped_normals_mask_max_dtc_value: 0.2
+env_normals_mask_max_dtc_value: 0.4
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: true
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type: small
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+baseline: false
+train_demo_idxs_list_or_num: -1
+log_train_every_steps: 25
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+store_dataset_in_memory: false
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+include_groups_list:
+- greece_twodim
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+name_of_valid_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr
+valid_demo_idxs_list_or_num: null
+val_num_groups: 0
+load_bc: ${agent.load_checkpoint}
+checkpoint_epoch_list:
+- 99
+- 199
+- 299
+- 399
+- 499
+- 599
+- 699
+- 799
+- 899
+- 999
+- 1249
+- 1499
+- 1749
+- 1999
+- 2999
+- 3999
+- 4999
+- 5999
+- 6999
+- 7999
+- 8999
+- 9999
+snapshot_root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data/FISH
+save_snapshot: true
+save_last_snapshot: true
+save_snapshot_when_done: true
+top_k_checkpoints: 5
+save_snapshot_link_to_weights_dir: deprecated
+bc_regularize: false
+bc_weight_type: qfilter
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: 6
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+      input_shapes:
+        observation.image:
+        - 13
+        - 180
+        - 240
+        context_observation.image:
+        - 13
+        - 180
+        - 240
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: cosine
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: 0.0001
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 1000000
+  num_seed_frames: 0
+  num_train_epochs: 15000
+  validate_every_epochs: 250
+  validate_diffusion_on_action_loss_every_epochs: 250
+  train_eval_diffusion_on_action_loss_every_epochs: 250
+  check_topk_every_epochs: 250
+  save_snapshot_every_epochs: 1500
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: ${device}
+      cam_resize_shape: ${agent.desired_image_shape}
+      orig_cam_shape: ${agent.orig_cam_shape}
+      policy_frequency: ${policy_frequency}
+      interpolation_frequency: ${interpolation_frequency}
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: ${agent.config.policy_cfg.horizon}
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: ${agent.config.policy_cfg.kernel_size}
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: ${num_train_frames_diffusion}
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: ${use_depth}
+        use_color: ${use_color}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_masks}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${color_crop_type}
+          depth_crop_type: ${depth_crop_type}
+          segmask_crop_type: ${segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${add_crop_binary_mask}
+          add_coord_conv_map: ${add_coord_conv_map}
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: ${use_context_color}
+          use_depth: ${use_context_depth}
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: ${use_context_segmask}
+            representation: ${mask_representation}
+            mask_list: ${mask_list}
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: ${context_color_crop_type}
+            depth_crop_type: ${context_depth_crop_type}
+            segmask_crop_type: ${context_segmask_crop_type}
+            crop_hw: ${crop_hw}
+            crop_down_offset: ${crop_down_offset}
+            add_crop_binary_mask: ${context_add_crop_binary_mask}
+            add_coord_conv_map: ${context_add_coord_conv_map}
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: ${use_contact_map}
+        use_sdf_maps: ${use_sdf_maps}
+        use_normals_maps: ${use_normals_maps}
+        which_objects: ${which_objects}
+        grasped_dtc_max_value: ${grasped_dtc_max_value}
+        env_dtc_max_value: ${env_dtc_max_value}
+        grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+        env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+        clamp_dtc: ${clamp_dtc}
+        max_contact_prob: ${max_contact_prob}
+        mask_normals_within_sdf: ${mask_normals_within_sdf}
+        dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+        adaptive_normals_mask: ${adaptive_normals_mask}
+        max_depth: ${max_depth}
+        image_shape: ${agent.desired_image_shape}
+        learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: ${contact_model_name}
+        zero_centered: false
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 15000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 250
+validate_diffusion_on_action_loss_every_epochs: 250
+train_eval_diffusion_on_action_loss_every_epochs: 250
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: 250
+save_snapshot_every_epochs_diffusion: 1500
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1