Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,18 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+205619/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+205619/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+205619/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+205619/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+205619/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/2_eval_2_eb964b6ad7f53c112f71.mp4 filter=lfs diff=lfs merge=lfs -text
+205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/3_eval_3_d1fd18c7f97afc0352cb.mp4 filter=lfs diff=lfs merge=lfs -text
+205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/4_eval_4_cc34baaf71385e46f645.mp4 filter=lfs diff=lfs merge=lfs -text
+205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/0_eval_0_1ac69680a9ec173c38e2.mp4 filter=lfs diff=lfs merge=lfs -text
+205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/1_eval_1_a59176de7339dfa56b5f.mp4 filter=lfs diff=lfs merge=lfs -text
+205619/episode_rosbags/episode_1_2024-12-17-20-57-23.bag filter=lfs diff=lfs merge=lfs -text
+205619/episode_rosbags/episode_2_2024-12-17-20-58-03.bag filter=lfs diff=lfs merge=lfs -text
+205619/episode_rosbags/episode_3_2024-12-17-20-58-48.bag filter=lfs diff=lfs merge=lfs -text
+205619/episode_rosbags/episode_4_2024-12-17-20-59-50.bag filter=lfs diff=lfs merge=lfs -text
+205619/episode_rosbags/episode_0_2024-12-17-20-56-42.bag filter=lfs diff=lfs merge=lfs -text

205617/.hydra/config.yaml

@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '506_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${now:%H%M%S}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

205617/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_policy
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205617
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

205617/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion

205617/eval_policy.log

@@ -0,0 +1,15 @@
+[2024-12-17 20:56:17,922][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-17 20:56:17,925][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2024-12-17 20:56:20,566][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+

205619/.hydra/config.yaml

@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '506_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${now:%H%M%S}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

205619/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

205619/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion

205619/episode_rosbags/aligned_depth_to_color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

205619/episode_rosbags/cam_tf_world.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256

205619/episode_rosbags/color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

205619/episode_rosbags/depth_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200

205619/episode_rosbags/episode_0_2024-12-17-20-56-42.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6db8d73088c48d573f729bd5a836c8e6ef5588a82a3e7f7d91875e9284179c56
+size 1333095236

205619/episode_rosbags/episode_1_2024-12-17-20-57-23.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b36e2c45975afe478240b83929742982642ef9cb6726ef46ca901f87f6ee3d4
+size 1336177279

205619/episode_rosbags/episode_2_2024-12-17-20-58-03.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c83cbcb7722bfa1d07ee32195bc7613660b3bcef41779295529c7bde79b3a674
+size 1330851491

205619/episode_rosbags/episode_3_2024-12-17-20-58-48.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:380f4d217077674246d3207c89778fddad576e1abc62a2bd20ac0f660b01eaa4
+size 1336822118

205619/episode_rosbags/episode_4_2024-12-17-20-59-50.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e75b1ebfa5532c1ec8d56e54057427aa095c1d98f86f56e5143312fe89abadb4
+size 1333619679

205619/eval_robot.log

@@ -0,0 +1,12 @@
+[2024-12-17 20:56:19,290][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-17 20:56:19,293][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+

205619/eval_video/0_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ac69680a9ec173c38e2f673f7cee924c41c969f46c49db4def6afddb959161a
+size 1130483

205619/eval_video/1_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a59176de7339dfa56b5f4aac2f1a34fa608ec2ce737cecc3b2c31a97af469c56
+size 1116795

205619/eval_video/2_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb964b6ad7f53c112f71de461af6718f919fd2aacd012218d2edc2ab8108dbc6
+size 1137506

205619/eval_video/3_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1fd18c7f97afc0352cbd0cc8abfff95a01692936410c447a7dfe7c209fbd3c3
+size 1139232

205619/eval_video/4_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc34baaf71385e46f645d4c9c51b86fab9aa8ad1520fd328bce80ad336b8289f
+size 1163338

205619/tb/events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fda363faa0814b06a04b823cea9331fd7004d69be288059789736a766f3be299
+size 1103

205619/wandb/debug-internal.log

@@ -0,0 +1,688 @@
+2024-12-17 20:56:25,183 INFO    StreamThr :70616 [internal.py:wandb_internal():85] W&B internal server running at pid: 70616, started at: 2024-12-17 20:56:25.182923
+2024-12-17 20:56:25,184 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status
+2024-12-17 20:56:25,193 INFO    WriterThread:70616 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/run-m7h1fbac.wandb
+2024-12-17 20:56:25,194 DEBUG   SenderThread:70616 [sender.py:send():379] send: header
+2024-12-17 20:56:25,206 DEBUG   SenderThread:70616 [sender.py:send():379] send: run
+2024-12-17 20:56:25,450 INFO    SenderThread:70616 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files
+2024-12-17 20:56:25,450 INFO    SenderThread:70616 [sender.py:_start_run_threads():1188] run started: m7h1fbac with start time 1734486985.183267
+2024-12-17 20:56:25,468 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: check_version
+2024-12-17 20:56:25,468 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: check_version
+2024-12-17 20:56:25,518 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: run_start
+2024-12-17 20:56:25,523 DEBUG   HandlerThread:70616 [system_info.py:__init__():26] System info init
+2024-12-17 20:56:25,523 DEBUG   HandlerThread:70616 [system_info.py:__init__():41] System info init done
+2024-12-17 20:56:25,523 INFO    HandlerThread:70616 [system_monitor.py:start():194] Starting system monitor
+2024-12-17 20:56:25,523 INFO    SystemMonitor:70616 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-17 20:56:25,523 INFO    HandlerThread:70616 [system_monitor.py:probe():214] Collecting system info
+2024-12-17 20:56:25,524 INFO    SystemMonitor:70616 [interfaces.py:start():188] Started cpu monitoring
+2024-12-17 20:56:25,524 INFO    SystemMonitor:70616 [interfaces.py:start():188] Started disk monitoring
+2024-12-17 20:56:25,525 INFO    SystemMonitor:70616 [interfaces.py:start():188] Started gpu monitoring
+2024-12-17 20:56:25,525 INFO    SystemMonitor:70616 [interfaces.py:start():188] Started memory monitoring
+2024-12-17 20:56:25,526 INFO    SystemMonitor:70616 [interfaces.py:start():188] Started network monitoring
+2024-12-17 20:56:25,537 DEBUG   HandlerThread:70616 [system_info.py:probe():152] Probing system
+2024-12-17 20:56:25,539 DEBUG   HandlerThread:70616 [system_info.py:_probe_git():137] Probing git
+2024-12-17 20:56:25,543 DEBUG   HandlerThread:70616 [system_info.py:_probe_git():145] Probing git done
+2024-12-17 20:56:25,543 DEBUG   HandlerThread:70616 [system_info.py:probe():200] Probing system done
+2024-12-17 20:56:25,543 DEBUG   HandlerThread:70616 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-18T01:56:25.537675', 'startedAt': '2024-12-18T01:56:25.175931', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '5f28b54373d7d41e4ebace380ff1080a8ffc4051'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3570.471375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.958, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 367.0489959716797}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2024-12-17 20:56:25,543 INFO    HandlerThread:70616 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-17 20:56:25,543 INFO    HandlerThread:70616 [system_monitor.py:probe():227] Publishing system info
+2024-12-17 20:56:25,544 DEBUG   HandlerThread:70616 [system_info.py:_save_code():44] Saving code
+2024-12-17 20:56:25,545 DEBUG   HandlerThread:70616 [system_info.py:_save_code():67] Saving code done
+2024-12-17 20:56:25,545 DEBUG   HandlerThread:70616 [system_info.py:_save_patches():84] Saving git patches
+2024-12-17 20:56:25,610 DEBUG   HandlerThread:70616 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-17 20:56:25,610 INFO    HandlerThread:70616 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-17 20:56:25,613 DEBUG   SenderThread:70616 [sender.py:send():379] send: files
+2024-12-17 20:56:25,613 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-17 20:56:25,613 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-17 20:56:25,614 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-17 20:56:25,843 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-17 20:56:25,843 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:56:25,843 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: python_packages
+2024-12-17 20:56:25,843 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:56:25,843 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:25,846 INFO    HandlerThread:70616 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/tb"
+  save: true
+}
+control {
+  relay_id: "19469a5384a5467b80ba8af284641d13"
+}
+_info {
+  stream_id: "m7h1fbac"
+}
+
+2024-12-17 20:56:25,846 DEBUG   HandlerThread:70616 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-17 20:56:25,873 INFO    wandb-upload_0:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/fe6nodd0-wandb-metadata.json
+2024-12-17 20:56:25,876 INFO    wandb-upload_2:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/yn4a2obv-diff.patch
+2024-12-17 20:56:25,907 INFO    wandb-upload_1:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/e3g49mjo-code/FISH/eval_robot.py
+2024-12-17 20:56:25,963 DEBUG   SenderThread:70616 [sender.py:send():379] send: telemetry
+2024-12-17 20:56:25,963 DEBUG   SenderThread:70616 [sender.py:send():379] send: tbrecord
+2024-12-17 20:56:25,964 DEBUG   SenderThread:70616 [sender.py:send():379] send: files
+2024-12-17 20:56:25,964 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0 with policy live
+2024-12-17 20:56:26,162 INFO    wandb-upload_0:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/wkffbmz1-events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0
+2024-12-17 20:56:26,451 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/code/FISH/eval_robot.py
+2024-12-17 20:56:26,452 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:26,452 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-metadata.json
+2024-12-17 20:56:26,452 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/requirements.txt
+2024-12-17 20:56:26,452 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/diff.patch
+2024-12-17 20:56:26,452 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0
+2024-12-17 20:56:26,452 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/code
+2024-12-17 20:56:26,452 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/code/FISH
+2024-12-17 20:56:26,843 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:27,843 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:28,451 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:28,843 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:29,700 DEBUG   SenderThread:70616 [sender.py:send():379] send: config
+2024-12-17 20:56:29,844 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:30,706 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:56:30,844 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:31,844 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:32,452 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:32,844 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:33,844 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:34,453 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:34,844 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:35,843 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:56:35,844 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:36,453 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:36,845 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:37,845 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:38,454 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:38,845 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:39,845 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:40,455 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:40,843 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:56:40,843 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:56:40,845 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:40,910 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:56:41,845 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:42,455 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:42,846 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:43,846 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:44,456 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:44,846 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:45,846 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:46,456 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:46,844 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:56:46,847 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:47,847 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:48,457 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:48,847 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:49,847 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:50,847 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:51,847 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:51,879 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:56:52,458 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:52,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:53,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:54,459 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:54,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:55,843 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:56:55,843 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:56:55,889 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:56,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:56,957 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:56:57,460 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/config.yaml
+2024-12-17 20:56:57,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:58,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:59,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:00,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:01,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:02,845 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:57:02,849 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:03,849 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:04,849 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:05,849 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:06,849 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:07,846 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:57:07,849 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:08,849 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:09,849 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:10,452 DEBUG   SenderThread:70616 [sender.py:send():379] send: files
+2024-12-17 20:57:10,452 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_1ac69680a9ec173c38e2.mp4 with policy now
+2024-12-17 20:57:10,452 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 20:57:10,453 DEBUG   SenderThread:70616 [sender.py:send():379] send: history
+2024-12-17 20:57:10,453 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:57:10,454 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:57:10,463 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:57:10,463 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/0_eval_0_1ac69680a9ec173c38e2.mp4
+2024-12-17 20:57:10,464 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json
+2024-12-17 20:57:10,464 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval
+2024-12-17 20:57:10,464 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos
+2024-12-17 20:57:10,464 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media
+2024-12-17 20:57:10,710 INFO    wandb-upload_1:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/2m43vjbj-media/videos/eval/0_eval_0_1ac69680a9ec173c38e2.mp4
+2024-12-17 20:57:10,843 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:57:10,844 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:57:10,849 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:10,971 INFO    wandb-upload_0:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/gxg5gj2f-media/videos/eval/0_eval_0_1ac69680a9ec173c38e2.mp4
+2024-12-17 20:57:11,853 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:12,464 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:57:12,850 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:12,984 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:57:13,851 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:14,852 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:15,852 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:16,465 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:57:16,852 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:17,852 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:18,466 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:57:18,821 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:57:18,852 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:19,466 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0
+2024-12-17 20:57:19,852 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:20,467 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:57:20,853 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:21,853 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:22,467 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:57:22,853 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:23,853 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:24,348 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:57:24,468 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:57:24,853 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:25,526 DEBUG   SystemMonitor:70616 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-17 20:57:25,526 DEBUG   SenderThread:70616 [sender.py:send():379] send: stats
+2024-12-17 20:57:25,843 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:57:25,844 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:57:25,885 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:26,854 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:27,854 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:28,854 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:29,854 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:29,966 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:57:30,854 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:31,855 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:32,855 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:33,855 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:34,855 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:35,846 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:57:35,855 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:36,855 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:37,855 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:38,855 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:39,855 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:40,844 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:57:40,844 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:57:40,889 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:40,932 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:57:41,855 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:42,856 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:43,856 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:44,856 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:45,856 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:46,847 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:57:46,856 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:47,856 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:48,856 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:49,856 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:50,856 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:51,053 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 20:57:51,054 DEBUG   SenderThread:70616 [sender.py:send():379] send: files
+2024-12-17 20:57:51,054 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_a59176de7339dfa56b5f.mp4 with policy now
+2024-12-17 20:57:51,054 DEBUG   SenderThread:70616 [sender.py:send():379] send: history
+2024-12-17 20:57:51,054 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:57:51,055 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:57:51,308 INFO    wandb-upload_1:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/gwsratzy-media/videos/eval/1_eval_1_a59176de7339dfa56b5f.mp4
+2024-12-17 20:57:51,476 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json
+2024-12-17 20:57:51,477 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/1_eval_1_a59176de7339dfa56b5f.mp4
+2024-12-17 20:57:51,477 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval
+2024-12-17 20:57:51,732 INFO    wandb-upload_0:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/00fbg6z1-media/videos/eval/1_eval_1_a59176de7339dfa56b5f.mp4
+2024-12-17 20:57:51,863 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:52,056 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:57:52,477 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:57:52,863 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:53,862 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:54,863 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:55,527 DEBUG   SenderThread:70616 [sender.py:send():379] send: stats
+2024-12-17 20:57:55,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:57:55,848 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:57:55,889 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:56,478 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:57:56,869 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:57,394 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:57:57,866 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:58,479 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:57:58,871 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:59,871 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:00,479 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0
+2024-12-17 20:58:00,723 INFO    wandb-upload_1:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/b5x3uf8q-events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0
+2024-12-17 20:58:00,867 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:01,239 DEBUG   SenderThread:70616 [sender.py:send():379] send: history
+2024-12-17 20:58:01,239 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:58:01,240 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:58:01,480 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json
+2024-12-17 20:58:01,868 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:02,480 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:02,873 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:03,241 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:58:03,868 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:04,868 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:05,868 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:06,481 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:06,868 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:07,868 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:08,482 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:08,868 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:09,049 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:58:09,868 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:10,849 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:58:10,849 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:58:10,869 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:11,869 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:12,484 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:12,869 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:13,870 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:14,485 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:14,870 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:15,049 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:58:15,870 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:16,485 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:16,871 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:17,871 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:18,870 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:19,872 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:20,487 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:20,849 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:58:20,872 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:21,872 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:22,872 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:23,872 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:24,488 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:24,872 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:25,527 DEBUG   SenderThread:70616 [sender.py:send():379] send: stats
+2024-12-17 20:58:25,844 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:58:25,845 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:58:25,885 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:25,903 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:58:26,872 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:27,873 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:28,873 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:29,873 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:30,873 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:31,398 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:58:31,654 DEBUG   SenderThread:70616 [sender.py:send():379] send: files
+2024-12-17 20:58:31,654 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_eb964b6ad7f53c112f71.mp4 with policy now
+2024-12-17 20:58:31,657 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 20:58:31,657 DEBUG   SenderThread:70616 [sender.py:send():379] send: history
+2024-12-17 20:58:31,657 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:58:31,658 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:58:31,874 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:31,907 INFO    wandb-upload_0:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/bz7uu82v-media/videos/eval/2_eval_2_eb964b6ad7f53c112f71.mp4
+2024-12-17 20:58:32,491 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:32,491 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json
+2024-12-17 20:58:32,491 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/2_eval_2_eb964b6ad7f53c112f71.mp4
+2024-12-17 20:58:32,491 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval
+2024-12-17 20:58:32,796 INFO    wandb-upload_1:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/u5o9rvmq-media/videos/eval/2_eval_2_eb964b6ad7f53c112f71.mp4
+2024-12-17 20:58:32,876 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:33,881 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:34,491 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:34,875 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:35,881 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:36,492 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:36,852 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:58:36,882 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:37,889 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:38,884 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:39,884 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:40,493 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:40,846 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:58:40,847 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:58:40,889 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:41,886 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:41,984 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:58:42,888 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:43,887 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:44,887 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:45,495 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0
+2024-12-17 20:58:45,893 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:46,061 DEBUG   SenderThread:70616 [sender.py:send():379] send: history
+2024-12-17 20:58:46,063 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:58:46,064 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:58:46,495 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:46,495 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json
+2024-12-17 20:58:46,890 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:47,225 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:58:47,891 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:48,496 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:48,891 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:49,890 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:50,497 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:50,890 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:51,890 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:52,852 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:58:52,890 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:53,890 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:54,890 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:55,528 DEBUG   SenderThread:70616 [sender.py:send():379] send: stats
+2024-12-17 20:58:55,845 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:58:55,845 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:58:55,891 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:56,891 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:57,891 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:57,920 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:58:58,891 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:59,891 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:00,891 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:01,891 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:02,892 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:03,852 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:59:03,892 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:04,892 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:05,892 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:06,892 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:07,893 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:08,853 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:59:08,894 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:09,893 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:10,845 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:59:10,845 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:59:10,893 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:11,893 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:12,504 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:59:12,893 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:13,893 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:13,931 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:59:14,894 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:15,894 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:16,505 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:59:16,871 DEBUG   SenderThread:70616 [sender.py:send():379] send: files
+2024-12-17 20:59:16,871 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_d1fd18c7f97afc0352cb.mp4 with policy now
+2024-12-17 20:59:16,879 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 20:59:16,880 DEBUG   SenderThread:70616 [sender.py:send():379] send: history
+2024-12-17 20:59:16,880 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:59:16,880 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:59:16,915 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:17,120 INFO    wandb-upload_0:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/zwvwl4qx-media/videos/eval/3_eval_3_d1fd18c7f97afc0352cb.mp4
+2024-12-17 20:59:17,506 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json
+2024-12-17 20:59:17,506 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/3_eval_3_d1fd18c7f97afc0352cb.mp4
+2024-12-17 20:59:17,506 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval
+2024-12-17 20:59:17,769 INFO    wandb-upload_1:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/5yfl7pkc-media/videos/eval/3_eval_3_d1fd18c7f97afc0352cb.mp4
+2024-12-17 20:59:17,916 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:18,506 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:59:18,904 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:19,892 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:59:19,925 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:20,917 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:21,912 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:22,507 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:59:22,912 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:23,914 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:24,508 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:59:24,919 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:25,373 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:59:25,528 DEBUG   SenderThread:70616 [sender.py:send():379] send: stats
+2024-12-17 20:59:25,850 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:59:25,850 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:59:25,917 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:26,916 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:27,920 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:28,923 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:29,924 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:30,870 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:59:30,919 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:31,920 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:32,927 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:33,928 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:34,924 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:35,923 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:36,867 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:59:36,921 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:37,922 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:38,929 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:39,931 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:40,863 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:59:40,863 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:59:40,926 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:41,929 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:41,989 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:59:42,931 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:43,941 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:44,932 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:45,942 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:46,515 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0
+2024-12-17 20:59:46,775 INFO    wandb-upload_0:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/4e06pnb5-events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0
+2024-12-17 20:59:46,935 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:47,138 DEBUG   SenderThread:70616 [sender.py:send():379] send: history
+2024-12-17 20:59:47,138 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:59:47,139 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:59:47,140 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:59:47,515 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json
+2024-12-17 20:59:47,932 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:48,516 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:59:48,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:49,933 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:50,932 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:51,933 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:52,517 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:59:52,865 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:59:52,933 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:53,933 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:54,933 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:55,529 DEBUG   SenderThread:70616 [sender.py:send():379] send: stats
+2024-12-17 20:59:55,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:59:55,848 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:59:55,933 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:56,933 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:57,933 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:57,987 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:59:58,933 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:59,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:00,933 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:01,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:02,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:03,865 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:00:03,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:04,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:05,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:06,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:07,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:08,522 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:08,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:08,979 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:00:09,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:10,523 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:10,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:00:10,848 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:00:10,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:11,935 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:12,935 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:13,935 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:14,525 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:14,779 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:00:14,935 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:15,935 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:16,525 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:16,937 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:17,953 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:18,526 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:18,693 DEBUG   SenderThread:70616 [sender.py:send():379] send: files
+2024-12-17 21:00:18,693 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_cc34baaf71385e46f645.mp4 with policy now
+2024-12-17 21:00:18,696 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:00:18,696 DEBUG   SenderThread:70616 [sender.py:send():379] send: history
+2024-12-17 21:00:18,696 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:00:18,696 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:00:18,952 INFO    wandb-upload_1:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/l2dpc4od-media/videos/eval/4_eval_4_cc34baaf71385e46f645.mp4
+2024-12-17 21:00:18,955 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:19,526 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json
+2024-12-17 21:00:19,527 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/4_eval_4_cc34baaf71385e46f645.mp4
+2024-12-17 21:00:19,527 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval
+2024-12-17 21:00:19,804 INFO    wandb-upload_0:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/c179dvnw-media/videos/eval/4_eval_4_cc34baaf71385e46f645.mp4
+2024-12-17 21:00:19,952 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:20,527 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:20,716 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:00:20,951 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:21,527 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:21,956 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:22,970 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:23,952 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:24,528 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:24,955 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:25,529 DEBUG   SenderThread:70616 [sender.py:send():379] send: stats
+2024-12-17 21:00:25,864 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:00:25,864 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:00:25,955 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:00:25,962 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:26,529 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:26,961 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:27,613 DEBUG   SenderThread:70616 [sender.py:send():379] send: exit
+2024-12-17 21:00:27,613 INFO    SenderThread:70616 [sender.py:send_exit():586] handling exit code: 0
+2024-12-17 21:00:27,613 INFO    SenderThread:70616 [sender.py:send_exit():588] handling runtime: 242
+2024-12-17 21:00:27,614 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:00:27,614 INFO    SenderThread:70616 [sender.py:send_exit():594] send defer
+2024-12-17 21:00:27,614 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:27,614 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-17 21:00:27,614 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:27,614 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-17 21:00:27,614 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 1
+2024-12-17 21:00:27,614 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:27,614 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-17 21:00:27,614 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:27,614 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-17 21:00:27,614 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 2
+2024-12-17 21:00:27,615 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:27,615 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-17 21:00:27,615 INFO    HandlerThread:70616 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-17 21:00:27,615 DEBUG   SystemMonitor:70616 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-17 21:00:27,615 DEBUG   SystemMonitor:70616 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-17 21:00:27,615 INFO    HandlerThread:70616 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-17 21:00:27,616 INFO    HandlerThread:70616 [interfaces.py:finish():200] Joined disk monitor
+2024-12-17 21:00:27,679 INFO    HandlerThread:70616 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-17 21:00:27,679 INFO    HandlerThread:70616 [interfaces.py:finish():200] Joined memory monitor
+2024-12-17 21:00:27,679 INFO    HandlerThread:70616 [interfaces.py:finish():200] Joined network monitor
+2024-12-17 21:00:27,680 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:27,680 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-17 21:00:27,680 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 3
+2024-12-17 21:00:27,680 DEBUG   SenderThread:70616 [sender.py:send():379] send: stats
+2024-12-17 21:00:27,680 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:27,680 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-17 21:00:27,680 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:27,680 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-17 21:00:27,680 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 4
+2024-12-17 21:00:27,680 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:27,680 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-17 21:00:28,530 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:28,530 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0
+2024-12-17 21:00:28,530 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json
+2024-12-17 21:00:33,682 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:00:33,683 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:00:33,683 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-17 21:00:33,683 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:33,683 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-17 21:00:33,683 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 5
+2024-12-17 21:00:33,683 DEBUG   SenderThread:70616 [sender.py:send():379] send: history
+2024-12-17 21:00:33,683 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:33,684 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:00:33,684 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-17 21:00:33,684 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:00:33,684 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:00:33,684 DEBUG   SenderThread:70616 [sender.py:send():379] send: history
+2024-12-17 21:00:33,685 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:00:33,688 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:00:33,688 DEBUG   SenderThread:70616 [sender.py:send():379] send: summary
+2024-12-17 21:00:33,689 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:00:33,689 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:33,689 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-17 21:00:33,689 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 6
+2024-12-17 21:00:33,689 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:33,689 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-17 21:00:33,689 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:33,689 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-17 21:00:33,689 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 7
+2024-12-17 21:00:33,690 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:00:33,690 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:33,690 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-17 21:00:33,690 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:33,690 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-17 21:00:34,532 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json
+2024-12-17 21:00:34,602 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 8
+2024-12-17 21:00:34,602 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:34,602 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-17 21:00:34,602 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:34,603 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-17 21:00:34,603 INFO    SenderThread:70616 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-17 21:00:34,605 INFO    SenderThread:70616 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-17 21:00:34,614 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:00:34,619 INFO    SenderThread:70616 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-17 21:00:34,632 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 9
+2024-12-17 21:00:34,632 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:00:34,632 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:34,632 DEBUG   SenderThread:70616 [sender.py:send():379] send: artifact
+2024-12-17 21:00:34,632 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-17 21:00:35,321 INFO    wandb-upload_2:70616 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpu4l34lz7
+2024-12-17 21:00:35,321 INFO    wandb-upload_0:70616 [upload_job.py:push():85] Skipped uploading /tmp/tmpthz7ujbr/wandb-job.json
+2024-12-17 21:00:35,452 INFO    wandb-upload_1:70616 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpouixkgus
+2024-12-17 21:00:35,533 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:35,615 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:00:36,051 INFO    SenderThread:70616 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM4MzA4NzA1NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM4MzA1OTA4Nw==', 'versionIndex': 374}}}
+2024-12-17 21:00:36,051 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:36,051 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-17 21:00:36,051 INFO    SenderThread:70616 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-17 21:00:36,534 INFO    SenderThread:70616 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files
+2024-12-17 21:00:36,534 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/config.yaml config.yaml
+2024-12-17 21:00:36,534 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json wandb-summary.json
+2024-12-17 21:00:36,534 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/requirements.txt requirements.txt
+2024-12-17 21:00:36,534 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-metadata.json wandb-metadata.json
+2024-12-17 21:00:36,537 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0 events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0
+2024-12-17 21:00:36,537 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log output.log
+2024-12-17 21:00:36,537 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/diff.patch diff.patch
+2024-12-17 21:00:36,539 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-17 21:00:36,539 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/0_eval_0_1ac69680a9ec173c38e2.mp4 media/videos/eval/0_eval_0_1ac69680a9ec173c38e2.mp4
+2024-12-17 21:00:36,539 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/4_eval_4_cc34baaf71385e46f645.mp4 media/videos/eval/4_eval_4_cc34baaf71385e46f645.mp4
+2024-12-17 21:00:36,540 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/2_eval_2_eb964b6ad7f53c112f71.mp4 media/videos/eval/2_eval_2_eb964b6ad7f53c112f71.mp4
+2024-12-17 21:00:36,541 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/3_eval_3_d1fd18c7f97afc0352cb.mp4 media/videos/eval/3_eval_3_d1fd18c7f97afc0352cb.mp4
+2024-12-17 21:00:36,541 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/1_eval_1_a59176de7339dfa56b5f.mp4 media/videos/eval/1_eval_1_a59176de7339dfa56b5f.mp4
+2024-12-17 21:00:36,542 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 10
+2024-12-17 21:00:36,542 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:00:36,543 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:36,543 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-17 21:00:36,543 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:36,543 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-17 21:00:36,543 INFO    SenderThread:70616 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 21:00:36,616 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:00:36,616 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:00:36,718 INFO    wandb-upload_0:70616 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json
+2024-12-17 21:00:36,731 INFO    wandb-upload_2:70616 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/config.yaml
+2024-12-17 21:00:37,118 INFO    wandb-upload_1:70616 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/requirements.txt
+2024-12-17 21:00:37,154 INFO    wandb-upload_3:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/igganeue-events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0
+2024-12-17 21:00:37,157 INFO    wandb-upload_4:70616 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:37,357 INFO    Thread-11 (_thread_body):70616 [sender.py:transition_state():614] send defer: 11
+2024-12-17 21:00:37,357 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:37,357 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-17 21:00:37,358 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:37,358 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-17 21:00:37,358 INFO    SenderThread:70616 [file_pusher.py:join():175] waiting for file pusher
+2024-12-17 21:00:37,359 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 12
+2024-12-17 21:00:37,359 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:37,359 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-17 21:00:37,359 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:37,359 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-17 21:00:37,359 INFO    SenderThread:70616 [file_stream.py:finish():601] file stream finish called
+2024-12-17 21:00:37,414 INFO    SenderThread:70616 [file_stream.py:finish():605] file stream finish is done
+2024-12-17 21:00:37,414 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 13
+2024-12-17 21:00:37,414 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:37,414 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-17 21:00:37,414 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:37,414 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-17 21:00:37,414 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 14
+2024-12-17 21:00:37,415 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:37,415 DEBUG   SenderThread:70616 [sender.py:send():379] send: final
+2024-12-17 21:00:37,415 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-17 21:00:37,415 DEBUG   SenderThread:70616 [sender.py:send():379] send: footer
+2024-12-17 21:00:37,415 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:37,415 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-17 21:00:37,415 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:00:37,415 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:00:37,416 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: server_info
+2024-12-17 21:00:37,416 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: server_info
+2024-12-17 21:00:37,417 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:00:37,417 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-17 21:00:37,418 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-17 21:00:37,418 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:37,497 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:00:37,497 INFO    MainThread:70616 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-17 21:00:37,497 INFO    MainThread:70616 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-17 21:00:37,497 INFO    MainThread:70616 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-17 21:00:37,498 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-17 21:00:37,498 INFO    HandlerThread:70616 [handler.py:finish():882] shutting down handler
+2024-12-17 21:00:38,418 INFO    WriterThread:70616 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/run-m7h1fbac.wandb
+2024-12-17 21:00:38,497 INFO    SenderThread:70616 [sender.py:finish():1615] shutting down sender
+2024-12-17 21:00:38,497 INFO    SenderThread:70616 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 21:00:38,497 INFO    SenderThread:70616 [file_pusher.py:join():175] waiting for file pusher

205619/wandb/debug.log

@@ -0,0 +1,31 @@
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_setup.py:_flush():76] Configure stats pid to 70432
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/settings
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/logs/debug.log
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/logs/debug-internal.log
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_init.py:init():569] calling init triggers
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '506_0', 'wandb_notes': '506_0_req_578_0', 'eval': True, 'true_action_history': False, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': False, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['lady_twodim'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/506_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619'}
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_init.py:init():619] starting backend
+2024-12-17 20:56:25,178 INFO    MainThread:70432 [wandb_init.py:init():623] setting up manager
+2024-12-17 20:56:25,182 INFO    MainThread:70432 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-17 20:56:25,182 INFO    MainThread:70432 [wandb_init.py:init():631] backend started and connected
+2024-12-17 20:56:25,193 INFO    MainThread:70432 [wandb_init.py:init():720] updated telemetry
+2024-12-17 20:56:25,203 INFO    MainThread:70432 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-17 20:56:25,467 INFO    MainThread:70432 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-17 20:56:25,514 INFO    MainThread:70432 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-17 20:56:25,514 INFO    MainThread:70432 [wandb_init.py:init():804] starting run threads in backend
+2024-12-17 20:56:25,843 INFO    MainThread:70432 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-17 20:56:25,843 INFO    MainThread:70432 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-17 20:56:25,843 INFO    MainThread:70432 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-17 20:56:25,844 INFO    MainThread:70432 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-17 20:56:25,845 INFO    MainThread:70432 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-17 20:56:25,845 INFO    MainThread:70432 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/tb, True
+2024-12-17 20:56:29,700 INFO    MainThread:70432 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'}
+2024-12-17 21:00:38,501 WARNING MsgRouterThr:70432 [router.py:message_loop():77] message_loop has been closed

205619/wandb/run-20241217_205625-m7h1fbac/files/code/FISH/eval_robot.py

@@ -0,0 +1,512 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)		
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()

205619/wandb/run-20241217_205625-m7h1fbac/files/config.yaml

@@ -0,0 +1,891 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 0
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '506_0'
+wandb_notes:
+  desc: null
+  value: 506_0_req_578_0
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: false
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: false
+use_sdf_maps:
+  desc: null
+  value: false
+use_normals_maps:
+  desc: null
+  value: false
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value:
+  - lady_twodim
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 0
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/kostas-graid/datasets/extrinsic_contact_data/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/506_0
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: false
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: false
+        use_sdf_maps: false
+        use_normals_maps: false
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 0
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: false
+          use_sdf_maps: false
+          use_normals_maps: false
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1734486985
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 16
+      - 23
+      - 35
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: lady
+left_book_slot:
+  desc: null
+  value: twodim

205619/wandb/run-20241217_205625-m7h1fbac/files/diff.patch

@@ -0,0 +1,195 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index bb0a413..15b3d90 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -107,7 +107,7 @@ num_valid_demos: null
+ val_num_groups: 3
+ 
+ # name_of_expert_demo: 'book_insertion_wo_ends_wedge_v2_76_all_demos_10hz_no_filter_wo_flow_mask_expert_demos_imp_act'
+-name_of_expert_demo: '192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act'
++name_of_expert_demo: '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act'
+ expert_dataset_dirpath: '${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}'
+ 
+ expert_dataset: '${expert_dataset_dirpath}/demos.zarr'
+@@ -123,89 +123,28 @@ bc_weight_type: 'qfilter' # linear, qfilter
+ 
+ # Load weights
+ load_checkpoint: ${agent.load_checkpoint}
+-# 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# full 32 demos
+-# wandb_run_id: "265625" # RGBD history seed 0
+-# wandb_run_id: "266370_1" # RGBD history seed 1
+-# wandb_run_id: "266370_2" # RGBD history seed 2
+-# wandb_run_id: "266382_4" # RGBD history seed 4
+-
+-# 16 held out demos (16 demos)
+-# wandb_run_id: "266392_0" # RGBD+act history dataset seed 0
+-# wandb_run_id: "266407_0" # RGBD+act history dataset seed 1
+-wandb_run_id: "266406_0" # RGBD+act history dataset seed 2
+-
+-# wandb_run_id: "265626" # RGBD+Mask+act history seed 0
+-# wandb_run_id: "265638" # RGBD+Mask+act history seed 1
+-# wandb_run_id: "265637" # RGBD+Mask+act history seed 2
+-# wandb_run_id: "265636" # RGBD+Mask+act history seed 3
+-# wandb_run_id: "265635" # RGBD+Mask+act history seed 4
+-
+-# wandb_run_id: "265628" # RGBD+Mask+contact+act history seed 0
+-# wandb_run_id: "265630" # RGBD+Mask+contact+act history seed 1
+-# wandb_run_id: "265631" # RGBD+Mask+contact+act history seed 2
+-# wandb_run_id: "265633" # RGBD+Mask+contact+act history seed 3
+-# wandb_run_id: "265634" # RGBD+Mask+contact+act history seed 4
+-
+-# 8 held out demos (26 demos)
+-# wandb_run_id: "265652" # RGBD+Mask+contact+act history dataset seed 0
+-# wandb_run_id: "265654" # RGBD+Mask+contact+act history dataset seed 1
+-# wandb_run_id: "265655" # RGBD+Mask+contact+act history dataset seed 2
+-
+-# 8 held out demos (26 demos)
+-# wandb_run_id: "265656" # RGBD+Mask+act history dataset seed 0
+-# wandb_run_id: "265657" # RGBD+Mask+act history dataset seed 1
+-# wandb_run_id: "265658" # RGBD+Mask+act history dataset seed 2
+-
+-# 16 held out demos (16 demos)
+-# wandb_run_id: "265641" # RGBD+Mask+act history dataset seed 0
+-# wandb_run_id: "265644" # RGBD+Mask+act history dataset seed 1
+-# wandb_run_id: "265645" # RGBD+Mask+act history dataset seed 2
+-# wandb_run_id: "266381_0" # RGBD+Mask+act history dataset seed 4
+-
+-# 16 held out demos (16 demos)
+-# wandb_run_id: "265646" # RGBD+Mask+Contact+act history dataset seed 0
+-# wandb_run_id: "265649" # RGBD+Mask+Contact+act history dataset seed 1
+-# wandb_run_id: "265650" # RGBD+Mask+Contact+act history dataset seed 2
+-# wandb_run_id: "266378_0" # RGBD+Mask+Contact+act history dataset seed 4
+-
+-
+-
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# full dataset
+-# 6 step action history
+-# wandb_run_id: '250358_0' # RGBD+act history
+-# wandb_run_id: '250371_0' # RGBD+mask+act history
+-# wandb_run_id: '250366_0' # RGBD+contact+act history
+-# wandb_run_id: '250376_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250366_0' # RGBD+contact+act history
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# 16 held out demos
+-# wandb_run_id: '250362_0' # RGBD+act history
+-# wandb_run_id: '250373_0' # RGBD+mask+act history
+-# wandb_run_id: '250378_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250368_0' # RGBD+contact+act history
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# 32 held out demos
+-# wandb_run_id: '250365_0' # RGBD+act history
+-# wandb_run_id: '250375_0' # RGBD+mask+act history
+-wandb_run_id: '250380_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250370_0' # RGBD+contact+act history
+-# 4 step action history
+-# wandb_run_id: '264772_0' # RGBD+act history
+-# wandb_run_id: '264775_0' # RGBD+mask+act history
+-# wandb_run_id: '264776_0' # RGBD+contact+act history
+-# wandb_run_id: '264777_0' # RGBD+mask+contact+act history
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# 16 held out demos
+-# wandb_run_id: '250362_0' # RGBD+act history
+-# wandb_run_id: '250373_0' # RGBD+mask+act history
+-# wandb_run_id: '250378_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250368_0' # RGBD+contact+act history
++# 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
++# 14 demos
++
++# RGBD+act history 
++# wandb_run_id: '497_0' # german
++# wandb_run_id: '505_0' # hbm
++wandb_run_id: '506_0' # lady
++# wandb_run_id: '507_0' # greece
++# wandb_run_id: '508_0' # fowlers
++# wandb_run_id: '509_0' # lib
++# wandb_run_id: '510_0' # modelsys
++# wandb_run_id: '511_0' # electrodyn
++
++# RGBD+mask+act history
++# wandb_run_id: '512_0' # german
++# wandb_run_id: '520_0' # hbm
++# wandb_run_id: '521_0' # lady
++# wandb_run_id: '522_0' # greece
++# wandb_run_id: '523_0' # fowlers
++# wandb_run_id: '524_0' # lib
++# wandb_run_id: '525_0' # modelsys
++# wandb_run_id: '526_0' # electrodyn
+ 
+ # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # 32 held out demos
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index 9406a46..0f225ec 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -15,12 +15,24 @@ if not local_checkpoint_root_dir.exists():
+     local_checkpoint_root_dir.mkdir(parents=True)
+ 
+ # Download the model checkpoints
+-remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1')
++# remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1')
++remote_checkpoint_root_dir = Path('/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1')
+ remote_checkpoint_root_dir = remote_checkpoint_root_dir.expanduser()
+ remote_username = 'leonmkim'
+-remote_host = 'halley'
++remote_host = 'grasp-login1'
+ 
+ #180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
++# run_id_list = [
++#     '512_0', # german
++#     '520_0', # hbm
++#     '521_0', # lady
++#     '522_0', # greece
++#     '523_0', # fowlers
++#     '524_0', # lib
++#     '525_0', # modelsys
++#     '526_0', # electrodyn
++# ]
++
+ # run_id_list = [
+ #     '265638',
+ #     '265637',
+@@ -75,6 +87,17 @@ remote_host = 'halley'
+ # ]
+ 
+ # 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
++# run_id_list = [
++#     '497_0', # german
++#     '505_0', # hbm
++#     '506_0', # lady
++#     '507_0', # greece
++#     '508_0', # fowlers
++#     '509_0', # lib
++#     '510_0', # modelsys
++#     '511_0', # electrodyn
++# ]
++
+ # run_id_list = [
+ #     '266382_4',
+ #     '266370_2',
+@@ -116,13 +139,13 @@ remote_host = 'halley'
+ # ]
+ 
+ # 180x240_1_RGB_D_2.0_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+-run_id_list = [
+-    # '266384_4,# not yet available
+-    '266372_1',
+-    '266372_2',
+-    '265627',
+-    '265620',
+-]
++# run_id_list = [
++#     # '266384_4,# not yet available
++#     '266372_1',
++#     '266372_2',
++#     '265627',
++#     '265620',
++# ]
+ checkpoint_epoch = 12000
+ 
+ # use subprocess to download the model checkpoints in parallel

205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/0_eval_0_1ac69680a9ec173c38e2.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ac69680a9ec173c38e2f673f7cee924c41c969f46c49db4def6afddb959161a
+size 1130483

205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/1_eval_1_a59176de7339dfa56b5f.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a59176de7339dfa56b5f4aac2f1a34fa608ec2ce737cecc3b2c31a97af469c56
+size 1116795

205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/2_eval_2_eb964b6ad7f53c112f71.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb964b6ad7f53c112f71de461af6718f919fd2aacd012218d2edc2ab8108dbc6
+size 1137506

205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/3_eval_3_d1fd18c7f97afc0352cb.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1fd18c7f97afc0352cbd0cc8abfff95a01692936410c447a7dfe7c209fbd3c3
+size 1139232

205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/4_eval_4_cc34baaf71385e46f645.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc34baaf71385e46f645d4c9c51b86fab9aa8ad1520fd328bce80ad336b8289f
+size 1163338

205619/wandb/run-20241217_205625-m7h1fbac/files/output.log

@@ -0,0 +1,238 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1734486989.700919]: resetting environment
+[INFO] [1734486989.702475]: cleared current plan
+[INFO] [1734486989.703508]: moving to home
+[INFO] [1734486990.705452]: reached home
+[INFO] [1734486990.707603]: reset action history
+[INFO] [1734486993.413871]: environment reset
+Starting episode 0
+[INFO] [1734486993.416165]: resetting environment
+[INFO] [1734486993.418828]: cleared current plan
+[INFO] [1734486993.420480]: moving to home
+[INFO] [1734486994.423058]: reached home
+[INFO] [1734486994.424734]: reset action history
+[INFO] [1734486997.131918]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1734486998.964343]: resetting environment
+[INFO] [1734486998.966101]: cleared current plan
+[INFO] [1734486998.967414]: moving to home
+[INFO] [1734486999.969260]: reached home
+[INFO] [1734486999.970628]: reset action history
+[INFO] [1734487002.677348]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/episode_rosbags/episode_0_2024-12-17-20-56-42.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-70432.log
+started roslaunch server http://158.130.50.37:44463/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [70713]
+process[print_text-2]: started with pid [70714]
+process[rosbag_record-3]: started with pid [70738]
+started bagging!
+For topic gripper_width: timestamp difference is -104593017 for nearest: 1734487005119791768 - target: 1734487005224384785 at idx 33
+For topic gripper_width: timestamp difference is -104180302 for nearest: 1734487005319792781 - target: 1734487005423973083 at idx 35
+For topic gripper_width: timestamp difference is -100442366 for nearest: 1734487007119796320 - target: 1734487007220238686 at idx 61
+For topic gripper_width: timestamp difference is -102225770 for nearest: 1734487009719793330 - target: 1734487009822019100 at idx 97
+For topic gripper_width: timestamp difference is -101047155 for nearest: 1734487011519791726 - target: 1734487011620838881 at idx 121
+For topic gripper_width: timestamp difference is -100888287 for nearest: 1734487011719792331 - target: 1734487011820680618 at idx 123
+[INFO] [1734487029.488217]: Storing episode...
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit
+[pub_text-1] killing on exit
+[INFO] [1734487030.222570]: Stored episode 1.
+[INFO] [1734487030.223587]: Saving video...
+[INFO] [1734487030.441256]: Video saved!
+Starting episode 1
+[INFO] [1734487030.458062]: resetting environment
+[INFO] [1734487030.458392]: cleared current plan
+[INFO] [1734487030.458826]: moving to home
+[INFO] [1734487034.062498]: reached home
+[INFO] [1734487034.065016]: reset action history
+[INFO] [1734487036.771729]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode![INFO] [1734487039.396081]: resetting environment
+[INFO] [1734487039.397059]: cleared current plan
+[INFO] [1734487039.397316]: moving to home
+[INFO] [1734487040.398803]: reached home
+[INFO] [1734487040.399214]: reset action history
+[INFO] [1734487043.105495]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/episode_rosbags/episode_1_2024-12-17-20-57-23.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-70432.log
+started roslaunch server http://158.130.50.37:35107/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [70841]
+process[print_text-5]: started with pid [70842]
+process[rosbag_record-6]: started with pid [70866]
+started bagging!
+[INFO] [1734487069.870051]: Storing episode...
+[rosbag_record-6] killing on exit
+[print_text-5] killing on exit
+[pub_text-4] killing on exit
+[INFO] [1734487070.621544]: Stored episode 2.
+[INFO] [1734487070.621784]: Saving video...
+[INFO] [1734487071.005145]: Video saved!
+Starting episode 2
+[INFO] [1734487071.036976]: resetting environment
+[INFO] [1734487071.038105]: cleared current plan
+[INFO] [1734487071.039962]: moving to home
+[INFO] [1734487074.649286]: reached home
+[INFO] [1734487074.650430]: reset action history
+[INFO] [1734487077.392814]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode![INFO] [1734487080.164351]: resetting environment
+[INFO] [1734487080.167381]: cleared current plan
+[INFO] [1734487080.168389]: moving to home
+[INFO] [1734487081.171356]: reached home
+[INFO] [1734487081.171792]: reset action history
+[INFO] [1734487083.881353]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/episode_rosbags/episode_2_2024-12-17-20-58-03.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-70432.log
+started roslaunch server http://158.130.50.37:43253/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [70969]
+process[print_text-8]: started with pid [70970]
+process[rosbag_record-9]: started with pid [70971]
+started bagging!
+For topic gripper_width: timestamp difference is -106642015 for nearest: 1734487086419790976 - target: 1734487086526432991 at idx 37
+For topic gripper_width: timestamp difference is -105802179 for nearest: 1734487086819789528 - target: 1734487086925591707 at idx 40
+For topic gripper_width: timestamp difference is -142847565 for nearest: 1734487090619817945 - target: 1734487090762665510 at idx 89
+For topic gripper_width: timestamp difference is -145199517 for nearest: 1734487091819790384 - target: 1734487091964989901 at idx 103
+For topic gripper_width: timestamp difference is -142071178 for nearest: 1734487093819791386 - target: 1734487093961862564 at idx 138
+For topic gripper_width: timestamp difference is -134369003 for nearest: 1734487097619790448 - target: 1734487097754159451 at idx 186
+For topic gripper_width: timestamp difference is -124582768 for nearest: 1734487102419790506 - target: 1734487102544373274 at idx 244
+[INFO] [1734487110.611957]: Storing episode...
+[rosbag_record-9] killing on exit
+[pub_text-7] killing on exit[print_text-8] killing on exit
+[INFO] [1734487111.398106]: Stored episode 3.
+[INFO] [1734487111.398558]: Saving video...
+[INFO] [1734487111.617893]: Video saved!
+Starting episode 3
+[INFO] [1734487111.658598]: resetting environment
+[INFO] [1734487111.661013]: cleared current plan
+[INFO] [1734487111.662566]: moving to home
+[INFO] [1734487115.269220]: reached home
+[INFO] [1734487115.275532]: reset action history
+[INFO] [1734487118.058185]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1734487125.171247]: resetting environment
+[INFO] [1734487125.173997]: cleared current plan
+[INFO] [1734487125.175082]: moving to home
+[INFO] [1734487126.177241]: reached home
+[INFO] [1734487126.182173]: reset action history
+[INFO] [1734487128.905090]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/episode_rosbags/episode_3_2024-12-17-20-58-48.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-70432.log
+started roslaunch server http://158.130.50.37:44327/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-10]: started with pid [71099]
+process[print_text-11]: started with pid [71100]
+process[rosbag_record-12]: started with pid [71101]
+started bagging!
+For topic gripper_width: timestamp difference is -255370055 for nearest: 1734487150319789495 - target: 1734487150575159550 at idx 249
+[INFO] [1734487155.641989]: Storing episode...
+[rosbag_record-12] killing on exit
+[pub_text-10] killing on exit
+[print_text-11] killing on exit
+[INFO] [1734487156.458836]: Stored episode 4.
+[INFO] [1734487156.461721]: Saving video...
+[INFO] [1734487156.820169]: Video saved!
+Starting episode 4
+[INFO] [1734487156.885524]: resetting environment
+[INFO] [1734487156.888706]: cleared current plan
+[INFO] [1734487156.891936]: moving to home
+[INFO] [1734487160.510341]: reached home
+[INFO] [1734487160.521129]: reset action history
+[INFO] [1734487163.323156]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1734487186.405382]: resetting environment
+[INFO] [1734487186.406476]: cleared current plan
+[INFO] [1734487186.414232]: moving to home
+[INFO] [1734487187.420798]: reached home
+[INFO] [1734487187.423901]: reset action history
+[INFO] [1734487190.199505]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/episode_rosbags/episode_4_2024-12-17-20-59-50.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-70432.log
+started roslaunch server http://158.130.50.37:36679/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-13]: started with pid [71254]
+process[print_text-14]: started with pid [71255]
+process[rosbag_record-15]: started with pid [71256]
+started bagging!
+For topic gripper_width: timestamp difference is -116999700 for nearest: 1734487206619788573 - target: 1734487206736788273 at idx 226
+For topic gripper_width: timestamp difference is -116582188 for nearest: 1734487206819791046 - target: 1734487206936373234 at idx 228
+For topic gripper_width: timestamp difference is -112480120 for nearest: 1734487208819791361 - target: 1734487208932271481 at idx 249
+For topic gripper_width: timestamp difference is -312054352 for nearest: 1734487208819791361 - target: 1734487209131845713 at idx 249
+For topic gripper_width: timestamp difference is -103390637 for nearest: 1734487213219788131 - target: 1734487213323178768 at idx 249
+For topic gripper_width: timestamp difference is -102970618 for nearest: 1734487213419815522 - target: 1734487213522786140 at idx 249
+For topic gripper_width: timestamp difference is -102632801 for nearest: 1734487213619755705 - target: 1734487213722388506 at idx 249
+[INFO] [1734487217.195205]: Storing episode...
+[rosbag_record-15] killing on exit
+[print_text-14] killing on exit
+[pub_text-13] killing on exit
+[INFO] [1734487218.024291]: Stored episode 5.
+[INFO] [1734487218.036450]: Saving video...
+[INFO] [1734487218.642226]: Video saved!
+[INFO] [1734487218.695607]: resetting environment
+[INFO] [1734487218.704827]: cleared current plan
+[INFO] [1734487218.715734]: moving to home
+[WARN] [1734487218.575874]: Plan exhausted
+[WARN] [1734487218.592690]: Plan exhausted
+[WARN] [1734487218.632129]: Plan exhausted
+[WARN] [1734487218.670600]: Plan exhausted
+[INFO] [1734487222.329906]: reached home
+[INFO] [1734487222.333436]: reset action history
+[INFO] [1734487225.133744]: environment reset
+Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!

205619/wandb/run-20241217_205625-m7h1fbac/files/requirements.txt

@@ -0,0 +1,339 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.2
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0

205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-metadata.json

@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2024-12-18T01:56:25.537675",
+    "startedAt": "2024-12-18T01:56:25.175931",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "5f28b54373d7d41e4ebace380ff1080a8ffc4051"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 3570.471375,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.958,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 367.0489959716797
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}

205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json

@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "1ac69680a9ec173c38e2f673f7cee924c41c969f46c49db4def6afddb959161a", "size": 1130483, "path": "media/videos/eval/0_eval_0_1ac69680a9ec173c38e2.mp4"}, "global_step": 660, "_timestamp": 1734487227.5709894, "_runtime": 242.38772225379944, "_step": 9, "eval/1_eval": {"_type": "video-file", "sha256": "a59176de7339dfa56b5f4aac2f1a34fa608ec2ce737cecc3b2c31a97af469c56", "size": 1116795, "path": "media/videos/eval/1_eval_1_a59176de7339dfa56b5f.mp4"}, "eval/num_success": 5.0, "episode": 5.0, "eval/success_rate": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "eb964b6ad7f53c112f71de461af6718f919fd2aacd012218d2edc2ab8108dbc6", "size": 1137506, "path": "media/videos/eval/2_eval_2_eb964b6ad7f53c112f71.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "d1fd18c7f97afc0352cbd0cc8abfff95a01692936410c447a7dfe7c209fbd3c3", "size": 1139232, "path": "media/videos/eval/3_eval_3_d1fd18c7f97afc0352cb.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "cc34baaf71385e46f645d4c9c51b86fab9aa8ad1520fd328bce80ad336b8289f", "size": 1163338, "path": "media/videos/eval/4_eval_4_cc34baaf71385e46f645.mp4"}, "_wandb": {"runtime": 242}}

205619/wandb/run-20241217_205625-m7h1fbac/logs/debug-internal.log

@@ -0,0 +1,688 @@
+2024-12-17 20:56:25,183 INFO    StreamThr :70616 [internal.py:wandb_internal():85] W&B internal server running at pid: 70616, started at: 2024-12-17 20:56:25.182923
+2024-12-17 20:56:25,184 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status
+2024-12-17 20:56:25,193 INFO    WriterThread:70616 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/run-m7h1fbac.wandb
+2024-12-17 20:56:25,194 DEBUG   SenderThread:70616 [sender.py:send():379] send: header
+2024-12-17 20:56:25,206 DEBUG   SenderThread:70616 [sender.py:send():379] send: run
+2024-12-17 20:56:25,450 INFO    SenderThread:70616 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files
+2024-12-17 20:56:25,450 INFO    SenderThread:70616 [sender.py:_start_run_threads():1188] run started: m7h1fbac with start time 1734486985.183267
+2024-12-17 20:56:25,468 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: check_version
+2024-12-17 20:56:25,468 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: check_version
+2024-12-17 20:56:25,518 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: run_start
+2024-12-17 20:56:25,523 DEBUG   HandlerThread:70616 [system_info.py:__init__():26] System info init
+2024-12-17 20:56:25,523 DEBUG   HandlerThread:70616 [system_info.py:__init__():41] System info init done
+2024-12-17 20:56:25,523 INFO    HandlerThread:70616 [system_monitor.py:start():194] Starting system monitor
+2024-12-17 20:56:25,523 INFO    SystemMonitor:70616 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-17 20:56:25,523 INFO    HandlerThread:70616 [system_monitor.py:probe():214] Collecting system info
+2024-12-17 20:56:25,524 INFO    SystemMonitor:70616 [interfaces.py:start():188] Started cpu monitoring
+2024-12-17 20:56:25,524 INFO    SystemMonitor:70616 [interfaces.py:start():188] Started disk monitoring
+2024-12-17 20:56:25,525 INFO    SystemMonitor:70616 [interfaces.py:start():188] Started gpu monitoring
+2024-12-17 20:56:25,525 INFO    SystemMonitor:70616 [interfaces.py:start():188] Started memory monitoring
+2024-12-17 20:56:25,526 INFO    SystemMonitor:70616 [interfaces.py:start():188] Started network monitoring
+2024-12-17 20:56:25,537 DEBUG   HandlerThread:70616 [system_info.py:probe():152] Probing system
+2024-12-17 20:56:25,539 DEBUG   HandlerThread:70616 [system_info.py:_probe_git():137] Probing git
+2024-12-17 20:56:25,543 DEBUG   HandlerThread:70616 [system_info.py:_probe_git():145] Probing git done
+2024-12-17 20:56:25,543 DEBUG   HandlerThread:70616 [system_info.py:probe():200] Probing system done
+2024-12-17 20:56:25,543 DEBUG   HandlerThread:70616 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-18T01:56:25.537675', 'startedAt': '2024-12-18T01:56:25.175931', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '5f28b54373d7d41e4ebace380ff1080a8ffc4051'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3570.471375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.958, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 367.0489959716797}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2024-12-17 20:56:25,543 INFO    HandlerThread:70616 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-17 20:56:25,543 INFO    HandlerThread:70616 [system_monitor.py:probe():227] Publishing system info
+2024-12-17 20:56:25,544 DEBUG   HandlerThread:70616 [system_info.py:_save_code():44] Saving code
+2024-12-17 20:56:25,545 DEBUG   HandlerThread:70616 [system_info.py:_save_code():67] Saving code done
+2024-12-17 20:56:25,545 DEBUG   HandlerThread:70616 [system_info.py:_save_patches():84] Saving git patches
+2024-12-17 20:56:25,610 DEBUG   HandlerThread:70616 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-17 20:56:25,610 INFO    HandlerThread:70616 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-17 20:56:25,613 DEBUG   SenderThread:70616 [sender.py:send():379] send: files
+2024-12-17 20:56:25,613 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-17 20:56:25,613 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-17 20:56:25,614 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-17 20:56:25,843 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-17 20:56:25,843 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:56:25,843 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: python_packages
+2024-12-17 20:56:25,843 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:56:25,843 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:25,846 INFO    HandlerThread:70616 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/tb"
+  save: true
+}
+control {
+  relay_id: "19469a5384a5467b80ba8af284641d13"
+}
+_info {
+  stream_id: "m7h1fbac"
+}
+
+2024-12-17 20:56:25,846 DEBUG   HandlerThread:70616 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-17 20:56:25,873 INFO    wandb-upload_0:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/fe6nodd0-wandb-metadata.json
+2024-12-17 20:56:25,876 INFO    wandb-upload_2:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/yn4a2obv-diff.patch
+2024-12-17 20:56:25,907 INFO    wandb-upload_1:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/e3g49mjo-code/FISH/eval_robot.py
+2024-12-17 20:56:25,963 DEBUG   SenderThread:70616 [sender.py:send():379] send: telemetry
+2024-12-17 20:56:25,963 DEBUG   SenderThread:70616 [sender.py:send():379] send: tbrecord
+2024-12-17 20:56:25,964 DEBUG   SenderThread:70616 [sender.py:send():379] send: files
+2024-12-17 20:56:25,964 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0 with policy live
+2024-12-17 20:56:26,162 INFO    wandb-upload_0:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/wkffbmz1-events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0
+2024-12-17 20:56:26,451 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/code/FISH/eval_robot.py
+2024-12-17 20:56:26,452 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:26,452 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-metadata.json
+2024-12-17 20:56:26,452 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/requirements.txt
+2024-12-17 20:56:26,452 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/diff.patch
+2024-12-17 20:56:26,452 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0
+2024-12-17 20:56:26,452 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/code
+2024-12-17 20:56:26,452 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/code/FISH
+2024-12-17 20:56:26,843 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:27,843 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:28,451 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:28,843 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:29,700 DEBUG   SenderThread:70616 [sender.py:send():379] send: config
+2024-12-17 20:56:29,844 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:30,706 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:56:30,844 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:31,844 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:32,452 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:32,844 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:33,844 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:34,453 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:34,844 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:35,843 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:56:35,844 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:36,453 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:36,845 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:37,845 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:38,454 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:38,845 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:39,845 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:40,455 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:40,843 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:56:40,843 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:56:40,845 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:40,910 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:56:41,845 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:42,455 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:42,846 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:43,846 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:44,456 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:44,846 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:45,846 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:46,456 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:46,844 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:56:46,847 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:47,847 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:48,457 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:48,847 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:49,847 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:50,847 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:51,847 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:51,879 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:56:52,458 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:52,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:53,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:54,459 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:56:54,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:55,843 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:56:55,843 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:56:55,889 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:56,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:56,957 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:56:57,460 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/config.yaml
+2024-12-17 20:56:57,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:58,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:56:59,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:00,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:01,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:02,845 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:57:02,849 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:03,849 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:04,849 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:05,849 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:06,849 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:07,846 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:57:07,849 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:08,849 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:09,849 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:10,452 DEBUG   SenderThread:70616 [sender.py:send():379] send: files
+2024-12-17 20:57:10,452 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_1ac69680a9ec173c38e2.mp4 with policy now
+2024-12-17 20:57:10,452 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 20:57:10,453 DEBUG   SenderThread:70616 [sender.py:send():379] send: history
+2024-12-17 20:57:10,453 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:57:10,454 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:57:10,463 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:57:10,463 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/0_eval_0_1ac69680a9ec173c38e2.mp4
+2024-12-17 20:57:10,464 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json
+2024-12-17 20:57:10,464 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval
+2024-12-17 20:57:10,464 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos
+2024-12-17 20:57:10,464 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media
+2024-12-17 20:57:10,710 INFO    wandb-upload_1:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/2m43vjbj-media/videos/eval/0_eval_0_1ac69680a9ec173c38e2.mp4
+2024-12-17 20:57:10,843 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:57:10,844 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:57:10,849 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:10,971 INFO    wandb-upload_0:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/gxg5gj2f-media/videos/eval/0_eval_0_1ac69680a9ec173c38e2.mp4
+2024-12-17 20:57:11,853 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:12,464 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:57:12,850 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:12,984 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:57:13,851 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:14,852 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:15,852 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:16,465 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:57:16,852 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:17,852 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:18,466 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:57:18,821 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:57:18,852 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:19,466 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0
+2024-12-17 20:57:19,852 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:20,467 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:57:20,853 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:21,853 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:22,467 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:57:22,853 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:23,853 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:24,348 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:57:24,468 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:57:24,853 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:25,526 DEBUG   SystemMonitor:70616 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-17 20:57:25,526 DEBUG   SenderThread:70616 [sender.py:send():379] send: stats
+2024-12-17 20:57:25,843 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:57:25,844 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:57:25,885 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:26,854 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:27,854 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:28,854 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:29,854 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:29,966 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:57:30,854 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:31,855 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:32,855 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:33,855 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:34,855 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:35,846 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:57:35,855 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:36,855 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:37,855 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:38,855 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:39,855 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:40,844 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:57:40,844 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:57:40,889 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:40,932 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:57:41,855 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:42,856 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:43,856 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:44,856 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:45,856 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:46,847 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:57:46,856 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:47,856 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:48,856 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:49,856 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:50,856 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:51,053 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 20:57:51,054 DEBUG   SenderThread:70616 [sender.py:send():379] send: files
+2024-12-17 20:57:51,054 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_a59176de7339dfa56b5f.mp4 with policy now
+2024-12-17 20:57:51,054 DEBUG   SenderThread:70616 [sender.py:send():379] send: history
+2024-12-17 20:57:51,054 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:57:51,055 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:57:51,308 INFO    wandb-upload_1:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/gwsratzy-media/videos/eval/1_eval_1_a59176de7339dfa56b5f.mp4
+2024-12-17 20:57:51,476 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json
+2024-12-17 20:57:51,477 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/1_eval_1_a59176de7339dfa56b5f.mp4
+2024-12-17 20:57:51,477 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval
+2024-12-17 20:57:51,732 INFO    wandb-upload_0:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/00fbg6z1-media/videos/eval/1_eval_1_a59176de7339dfa56b5f.mp4
+2024-12-17 20:57:51,863 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:52,056 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:57:52,477 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:57:52,863 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:53,862 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:54,863 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:55,527 DEBUG   SenderThread:70616 [sender.py:send():379] send: stats
+2024-12-17 20:57:55,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:57:55,848 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:57:55,889 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:56,478 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:57:56,869 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:57,394 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:57:57,866 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:58,479 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:57:58,871 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:57:59,871 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:00,479 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0
+2024-12-17 20:58:00,723 INFO    wandb-upload_1:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/b5x3uf8q-events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0
+2024-12-17 20:58:00,867 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:01,239 DEBUG   SenderThread:70616 [sender.py:send():379] send: history
+2024-12-17 20:58:01,239 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:58:01,240 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:58:01,480 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json
+2024-12-17 20:58:01,868 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:02,480 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:02,873 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:03,241 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:58:03,868 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:04,868 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:05,868 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:06,481 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:06,868 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:07,868 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:08,482 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:08,868 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:09,049 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:58:09,868 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:10,849 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:58:10,849 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:58:10,869 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:11,869 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:12,484 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:12,869 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:13,870 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:14,485 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:14,870 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:15,049 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:58:15,870 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:16,485 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:16,871 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:17,871 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:18,870 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:19,872 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:20,487 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:20,849 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:58:20,872 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:21,872 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:22,872 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:23,872 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:24,488 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:24,872 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:25,527 DEBUG   SenderThread:70616 [sender.py:send():379] send: stats
+2024-12-17 20:58:25,844 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:58:25,845 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:58:25,885 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:25,903 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:58:26,872 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:27,873 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:28,873 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:29,873 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:30,873 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:31,398 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:58:31,654 DEBUG   SenderThread:70616 [sender.py:send():379] send: files
+2024-12-17 20:58:31,654 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_eb964b6ad7f53c112f71.mp4 with policy now
+2024-12-17 20:58:31,657 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 20:58:31,657 DEBUG   SenderThread:70616 [sender.py:send():379] send: history
+2024-12-17 20:58:31,657 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:58:31,658 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:58:31,874 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:31,907 INFO    wandb-upload_0:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/bz7uu82v-media/videos/eval/2_eval_2_eb964b6ad7f53c112f71.mp4
+2024-12-17 20:58:32,491 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:32,491 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json
+2024-12-17 20:58:32,491 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/2_eval_2_eb964b6ad7f53c112f71.mp4
+2024-12-17 20:58:32,491 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval
+2024-12-17 20:58:32,796 INFO    wandb-upload_1:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/u5o9rvmq-media/videos/eval/2_eval_2_eb964b6ad7f53c112f71.mp4
+2024-12-17 20:58:32,876 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:33,881 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:34,491 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:34,875 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:35,881 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:36,492 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:36,852 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:58:36,882 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:37,889 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:38,884 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:39,884 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:40,493 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:40,846 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:58:40,847 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:58:40,889 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:41,886 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:41,984 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:58:42,888 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:43,887 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:44,887 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:45,495 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0
+2024-12-17 20:58:45,893 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:46,061 DEBUG   SenderThread:70616 [sender.py:send():379] send: history
+2024-12-17 20:58:46,063 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:58:46,064 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:58:46,495 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:46,495 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json
+2024-12-17 20:58:46,890 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:47,225 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:58:47,891 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:48,496 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:48,891 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:49,890 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:50,497 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:58:50,890 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:51,890 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:52,852 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:58:52,890 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:53,890 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:54,890 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:55,528 DEBUG   SenderThread:70616 [sender.py:send():379] send: stats
+2024-12-17 20:58:55,845 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:58:55,845 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:58:55,891 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:56,891 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:57,891 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:57,920 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:58:58,891 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:58:59,891 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:00,891 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:01,891 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:02,892 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:03,852 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:59:03,892 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:04,892 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:05,892 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:06,892 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:07,893 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:08,853 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:59:08,894 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:09,893 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:10,845 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:59:10,845 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:59:10,893 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:11,893 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:12,504 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:59:12,893 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:13,893 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:13,931 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:59:14,894 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:15,894 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:16,505 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:59:16,871 DEBUG   SenderThread:70616 [sender.py:send():379] send: files
+2024-12-17 20:59:16,871 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_d1fd18c7f97afc0352cb.mp4 with policy now
+2024-12-17 20:59:16,879 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 20:59:16,880 DEBUG   SenderThread:70616 [sender.py:send():379] send: history
+2024-12-17 20:59:16,880 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:59:16,880 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:59:16,915 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:17,120 INFO    wandb-upload_0:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/zwvwl4qx-media/videos/eval/3_eval_3_d1fd18c7f97afc0352cb.mp4
+2024-12-17 20:59:17,506 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json
+2024-12-17 20:59:17,506 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/3_eval_3_d1fd18c7f97afc0352cb.mp4
+2024-12-17 20:59:17,506 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval
+2024-12-17 20:59:17,769 INFO    wandb-upload_1:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/5yfl7pkc-media/videos/eval/3_eval_3_d1fd18c7f97afc0352cb.mp4
+2024-12-17 20:59:17,916 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:18,506 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:59:18,904 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:19,892 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:59:19,925 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:20,917 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:21,912 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:22,507 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:59:22,912 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:23,914 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:24,508 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:59:24,919 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:25,373 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:59:25,528 DEBUG   SenderThread:70616 [sender.py:send():379] send: stats
+2024-12-17 20:59:25,850 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:59:25,850 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:59:25,917 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:26,916 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:27,920 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:28,923 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:29,924 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:30,870 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:59:30,919 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:31,920 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:32,927 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:33,928 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:34,924 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:35,923 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:36,867 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:59:36,921 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:37,922 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:38,929 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:39,931 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:40,863 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:59:40,863 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:59:40,926 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:41,929 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:41,989 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:59:42,931 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:43,941 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:44,932 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:45,942 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:46,515 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0
+2024-12-17 20:59:46,775 INFO    wandb-upload_0:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/4e06pnb5-events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0
+2024-12-17 20:59:46,935 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:47,138 DEBUG   SenderThread:70616 [sender.py:send():379] send: history
+2024-12-17 20:59:47,138 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 20:59:47,139 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:59:47,140 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 20:59:47,515 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json
+2024-12-17 20:59:47,932 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:48,516 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:59:48,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:49,933 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:50,932 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:51,933 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:52,517 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 20:59:52,865 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:59:52,933 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:53,933 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:54,933 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:55,529 DEBUG   SenderThread:70616 [sender.py:send():379] send: stats
+2024-12-17 20:59:55,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 20:59:55,848 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 20:59:55,933 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:56,933 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:57,933 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:57,987 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 20:59:58,933 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 20:59:59,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:00,933 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:01,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:02,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:03,865 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:00:03,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:04,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:05,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:06,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:07,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:08,522 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:08,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:08,979 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:00:09,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:10,523 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:10,848 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:00:10,848 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:00:10,934 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:11,935 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:12,935 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:13,935 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:14,525 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:14,779 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:00:14,935 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:15,935 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:16,525 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:16,937 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:17,953 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:18,526 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:18,693 DEBUG   SenderThread:70616 [sender.py:send():379] send: files
+2024-12-17 21:00:18,693 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_cc34baaf71385e46f645.mp4 with policy now
+2024-12-17 21:00:18,696 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 21:00:18,696 DEBUG   SenderThread:70616 [sender.py:send():379] send: history
+2024-12-17 21:00:18,696 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:00:18,696 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:00:18,952 INFO    wandb-upload_1:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/l2dpc4od-media/videos/eval/4_eval_4_cc34baaf71385e46f645.mp4
+2024-12-17 21:00:18,955 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:19,526 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json
+2024-12-17 21:00:19,527 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/4_eval_4_cc34baaf71385e46f645.mp4
+2024-12-17 21:00:19,527 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval
+2024-12-17 21:00:19,804 INFO    wandb-upload_0:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/c179dvnw-media/videos/eval/4_eval_4_cc34baaf71385e46f645.mp4
+2024-12-17 21:00:19,952 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:20,527 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:20,716 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:00:20,951 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:21,527 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:21,956 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:22,970 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:23,952 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:24,528 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:24,955 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:25,529 DEBUG   SenderThread:70616 [sender.py:send():379] send: stats
+2024-12-17 21:00:25,864 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 21:00:25,864 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 21:00:25,955 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:00:25,962 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:26,529 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:26,961 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:27,613 DEBUG   SenderThread:70616 [sender.py:send():379] send: exit
+2024-12-17 21:00:27,613 INFO    SenderThread:70616 [sender.py:send_exit():586] handling exit code: 0
+2024-12-17 21:00:27,613 INFO    SenderThread:70616 [sender.py:send_exit():588] handling runtime: 242
+2024-12-17 21:00:27,614 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:00:27,614 INFO    SenderThread:70616 [sender.py:send_exit():594] send defer
+2024-12-17 21:00:27,614 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:27,614 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-17 21:00:27,614 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:27,614 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-17 21:00:27,614 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 1
+2024-12-17 21:00:27,614 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:27,614 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-17 21:00:27,614 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:27,614 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-17 21:00:27,614 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 2
+2024-12-17 21:00:27,615 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:27,615 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-17 21:00:27,615 INFO    HandlerThread:70616 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-17 21:00:27,615 DEBUG   SystemMonitor:70616 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-17 21:00:27,615 DEBUG   SystemMonitor:70616 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-17 21:00:27,615 INFO    HandlerThread:70616 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-17 21:00:27,616 INFO    HandlerThread:70616 [interfaces.py:finish():200] Joined disk monitor
+2024-12-17 21:00:27,679 INFO    HandlerThread:70616 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-17 21:00:27,679 INFO    HandlerThread:70616 [interfaces.py:finish():200] Joined memory monitor
+2024-12-17 21:00:27,679 INFO    HandlerThread:70616 [interfaces.py:finish():200] Joined network monitor
+2024-12-17 21:00:27,680 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:27,680 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-17 21:00:27,680 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 3
+2024-12-17 21:00:27,680 DEBUG   SenderThread:70616 [sender.py:send():379] send: stats
+2024-12-17 21:00:27,680 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:27,680 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-17 21:00:27,680 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:27,680 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-17 21:00:27,680 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 4
+2024-12-17 21:00:27,680 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:27,680 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-17 21:00:28,530 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:28,530 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0
+2024-12-17 21:00:28,530 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json
+2024-12-17 21:00:33,682 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:00:33,683 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:00:33,683 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-17 21:00:33,683 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:33,683 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-17 21:00:33,683 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 5
+2024-12-17 21:00:33,683 DEBUG   SenderThread:70616 [sender.py:send():379] send: history
+2024-12-17 21:00:33,683 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:33,684 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:00:33,684 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-17 21:00:33,684 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:00:33,684 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:00:33,684 DEBUG   SenderThread:70616 [sender.py:send():379] send: history
+2024-12-17 21:00:33,685 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 21:00:33,688 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:00:33,688 DEBUG   SenderThread:70616 [sender.py:send():379] send: summary
+2024-12-17 21:00:33,689 INFO    SenderThread:70616 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 21:00:33,689 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:33,689 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-17 21:00:33,689 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 6
+2024-12-17 21:00:33,689 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:33,689 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-17 21:00:33,689 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:33,689 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-17 21:00:33,689 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 7
+2024-12-17 21:00:33,690 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 21:00:33,690 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:33,690 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-17 21:00:33,690 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:33,690 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-17 21:00:34,532 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json
+2024-12-17 21:00:34,602 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 8
+2024-12-17 21:00:34,602 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:34,602 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-17 21:00:34,602 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:34,603 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-17 21:00:34,603 INFO    SenderThread:70616 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-17 21:00:34,605 INFO    SenderThread:70616 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-17 21:00:34,614 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:00:34,619 INFO    SenderThread:70616 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-17 21:00:34,632 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 9
+2024-12-17 21:00:34,632 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:00:34,632 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:34,632 DEBUG   SenderThread:70616 [sender.py:send():379] send: artifact
+2024-12-17 21:00:34,632 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-17 21:00:35,321 INFO    wandb-upload_2:70616 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpu4l34lz7
+2024-12-17 21:00:35,321 INFO    wandb-upload_0:70616 [upload_job.py:push():85] Skipped uploading /tmp/tmpthz7ujbr/wandb-job.json
+2024-12-17 21:00:35,452 INFO    wandb-upload_1:70616 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpouixkgus
+2024-12-17 21:00:35,533 INFO    Thread-12 :70616 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:35,615 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:00:36,051 INFO    SenderThread:70616 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM4MzA4NzA1NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM4MzA1OTA4Nw==', 'versionIndex': 374}}}
+2024-12-17 21:00:36,051 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:36,051 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-17 21:00:36,051 INFO    SenderThread:70616 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-17 21:00:36,534 INFO    SenderThread:70616 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files
+2024-12-17 21:00:36,534 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/config.yaml config.yaml
+2024-12-17 21:00:36,534 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json wandb-summary.json
+2024-12-17 21:00:36,534 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/requirements.txt requirements.txt
+2024-12-17 21:00:36,534 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-metadata.json wandb-metadata.json
+2024-12-17 21:00:36,537 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0 events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0
+2024-12-17 21:00:36,537 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log output.log
+2024-12-17 21:00:36,537 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/diff.patch diff.patch
+2024-12-17 21:00:36,539 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-17 21:00:36,539 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/0_eval_0_1ac69680a9ec173c38e2.mp4 media/videos/eval/0_eval_0_1ac69680a9ec173c38e2.mp4
+2024-12-17 21:00:36,539 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/4_eval_4_cc34baaf71385e46f645.mp4 media/videos/eval/4_eval_4_cc34baaf71385e46f645.mp4
+2024-12-17 21:00:36,540 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/2_eval_2_eb964b6ad7f53c112f71.mp4 media/videos/eval/2_eval_2_eb964b6ad7f53c112f71.mp4
+2024-12-17 21:00:36,541 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/3_eval_3_d1fd18c7f97afc0352cb.mp4 media/videos/eval/3_eval_3_d1fd18c7f97afc0352cb.mp4
+2024-12-17 21:00:36,541 INFO    SenderThread:70616 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/media/videos/eval/1_eval_1_a59176de7339dfa56b5f.mp4 media/videos/eval/1_eval_1_a59176de7339dfa56b5f.mp4
+2024-12-17 21:00:36,542 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 10
+2024-12-17 21:00:36,542 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:00:36,543 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:36,543 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-17 21:00:36,543 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:36,543 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-17 21:00:36,543 INFO    SenderThread:70616 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 21:00:36,616 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:00:36,616 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:00:36,718 INFO    wandb-upload_0:70616 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/wandb-summary.json
+2024-12-17 21:00:36,731 INFO    wandb-upload_2:70616 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/config.yaml
+2024-12-17 21:00:37,118 INFO    wandb-upload_1:70616 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/requirements.txt
+2024-12-17 21:00:37,154 INFO    wandb-upload_3:70616 [upload_job.py:push():130] Uploaded file /tmp/tmpd0e87ne3wandb/igganeue-events.out.tfevents.1734486985.leonmkim-ROG-Strix-G15CS-G15CS.70432.0
+2024-12-17 21:00:37,157 INFO    wandb-upload_4:70616 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/files/output.log
+2024-12-17 21:00:37,357 INFO    Thread-11 (_thread_body):70616 [sender.py:transition_state():614] send defer: 11
+2024-12-17 21:00:37,357 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:37,357 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-17 21:00:37,358 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:37,358 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-17 21:00:37,358 INFO    SenderThread:70616 [file_pusher.py:join():175] waiting for file pusher
+2024-12-17 21:00:37,359 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 12
+2024-12-17 21:00:37,359 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:37,359 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-17 21:00:37,359 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:37,359 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-17 21:00:37,359 INFO    SenderThread:70616 [file_stream.py:finish():601] file stream finish called
+2024-12-17 21:00:37,414 INFO    SenderThread:70616 [file_stream.py:finish():605] file stream finish is done
+2024-12-17 21:00:37,414 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 13
+2024-12-17 21:00:37,414 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:37,414 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-17 21:00:37,414 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:37,414 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-17 21:00:37,414 INFO    SenderThread:70616 [sender.py:transition_state():614] send defer: 14
+2024-12-17 21:00:37,415 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 21:00:37,415 DEBUG   SenderThread:70616 [sender.py:send():379] send: final
+2024-12-17 21:00:37,415 INFO    HandlerThread:70616 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-17 21:00:37,415 DEBUG   SenderThread:70616 [sender.py:send():379] send: footer
+2024-12-17 21:00:37,415 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: defer
+2024-12-17 21:00:37,415 INFO    SenderThread:70616 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-17 21:00:37,415 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:00:37,415 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:00:37,416 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: server_info
+2024-12-17 21:00:37,416 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: server_info
+2024-12-17 21:00:37,417 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 21:00:37,417 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-17 21:00:37,418 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-17 21:00:37,418 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 21:00:37,497 DEBUG   SenderThread:70616 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 21:00:37,497 INFO    MainThread:70616 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-17 21:00:37,497 INFO    MainThread:70616 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-17 21:00:37,497 INFO    MainThread:70616 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-17 21:00:37,498 DEBUG   HandlerThread:70616 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-17 21:00:37,498 INFO    HandlerThread:70616 [handler.py:finish():882] shutting down handler
+2024-12-17 21:00:38,418 INFO    WriterThread:70616 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/run-m7h1fbac.wandb
+2024-12-17 21:00:38,497 INFO    SenderThread:70616 [sender.py:finish():1615] shutting down sender
+2024-12-17 21:00:38,497 INFO    SenderThread:70616 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 21:00:38,497 INFO    SenderThread:70616 [file_pusher.py:join():175] waiting for file pusher

205619/wandb/run-20241217_205625-m7h1fbac/logs/debug.log

@@ -0,0 +1,31 @@
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_setup.py:_flush():76] Configure stats pid to 70432
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/settings
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/logs/debug.log
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/wandb/run-20241217_205625-m7h1fbac/logs/debug-internal.log
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_init.py:init():569] calling init triggers
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '506_0', 'wandb_notes': '506_0_req_578_0', 'eval': True, 'true_action_history': False, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': False, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['lady_twodim'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/506_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619'}
+2024-12-17 20:56:25,177 INFO    MainThread:70432 [wandb_init.py:init():619] starting backend
+2024-12-17 20:56:25,178 INFO    MainThread:70432 [wandb_init.py:init():623] setting up manager
+2024-12-17 20:56:25,182 INFO    MainThread:70432 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-17 20:56:25,182 INFO    MainThread:70432 [wandb_init.py:init():631] backend started and connected
+2024-12-17 20:56:25,193 INFO    MainThread:70432 [wandb_init.py:init():720] updated telemetry
+2024-12-17 20:56:25,203 INFO    MainThread:70432 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-17 20:56:25,467 INFO    MainThread:70432 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-17 20:56:25,514 INFO    MainThread:70432 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-17 20:56:25,514 INFO    MainThread:70432 [wandb_init.py:init():804] starting run threads in backend
+2024-12-17 20:56:25,843 INFO    MainThread:70432 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-17 20:56:25,843 INFO    MainThread:70432 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-17 20:56:25,843 INFO    MainThread:70432 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-17 20:56:25,844 INFO    MainThread:70432 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-17 20:56:25,845 INFO    MainThread:70432 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-17 20:56:25,845 INFO    MainThread:70432 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/506_0/205619/tb, True
+2024-12-17 20:56:29,700 INFO    MainThread:70432 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'}
+2024-12-17 21:00:38,501 WARNING MsgRouterThr:70432 [router.py:message_loop():77] message_loop has been closed

config.yaml

@@ -0,0 +1,546 @@
+root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data
+replay_buffer_size: 150000
+replay_buffer_num_workers: 2
+nstep: 3
+batch_size: 128
+seed: 0
+dataset_shuffle_seed: 0
+device: cuda
+save_video: true
+save_train_video: true
+use_tb: true
+use_wandb: true
+wandb_run_id: '506_0'
+wandb_notes: 506_0_req_578_0
+eval: false
+true_action_history: false
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.2
+env_dtc_max_value: 0.4
+grasped_normals_mask_max_dtc_value: 0.2
+env_normals_mask_max_dtc_value: 0.4
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: true
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type: small
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+baseline: false
+train_demo_idxs_list_or_num: -1
+log_train_every_steps: 25
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+store_dataset_in_memory: false
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+include_groups_list:
+- lady_twodim
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+name_of_valid_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr
+valid_demo_idxs_list_or_num: null
+val_num_groups: 0
+load_bc: ${agent.load_checkpoint}
+checkpoint_epoch_list:
+- 99
+- 199
+- 299
+- 399
+- 499
+- 599
+- 699
+- 799
+- 899
+- 999
+- 1249
+- 1499
+- 1749
+- 1999
+- 2999
+- 3999
+- 4999
+- 5999
+- 6999
+- 7999
+- 8999
+- 9999
+snapshot_root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data/FISH
+save_snapshot: true
+save_last_snapshot: true
+save_snapshot_when_done: true
+top_k_checkpoints: 5
+save_snapshot_link_to_weights_dir: deprecated
+bc_regularize: false
+bc_weight_type: qfilter
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: 6
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+      input_shapes:
+        observation.image:
+        - 13
+        - 180
+        - 240
+        context_observation.image:
+        - 13
+        - 180
+        - 240
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: cosine
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: 0.0001
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 1000000
+  num_seed_frames: 0
+  num_train_epochs: 15000
+  validate_every_epochs: 250
+  validate_diffusion_on_action_loss_every_epochs: 250
+  train_eval_diffusion_on_action_loss_every_epochs: 250
+  check_topk_every_epochs: 250
+  save_snapshot_every_epochs: 1500
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: ${device}
+      cam_resize_shape: ${agent.desired_image_shape}
+      orig_cam_shape: ${agent.orig_cam_shape}
+      policy_frequency: ${policy_frequency}
+      interpolation_frequency: ${interpolation_frequency}
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: ${agent.config.policy_cfg.horizon}
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: ${agent.config.policy_cfg.kernel_size}
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: ${num_train_frames_diffusion}
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: ${use_depth}
+        use_color: ${use_color}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_masks}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${color_crop_type}
+          depth_crop_type: ${depth_crop_type}
+          segmask_crop_type: ${segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${add_crop_binary_mask}
+          add_coord_conv_map: ${add_coord_conv_map}
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: ${use_context_color}
+          use_depth: ${use_context_depth}
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: ${use_context_segmask}
+            representation: ${mask_representation}
+            mask_list: ${mask_list}
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: ${context_color_crop_type}
+            depth_crop_type: ${context_depth_crop_type}
+            segmask_crop_type: ${context_segmask_crop_type}
+            crop_hw: ${crop_hw}
+            crop_down_offset: ${crop_down_offset}
+            add_crop_binary_mask: ${context_add_crop_binary_mask}
+            add_coord_conv_map: ${context_add_coord_conv_map}
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: ${use_contact_map}
+        use_sdf_maps: ${use_sdf_maps}
+        use_normals_maps: ${use_normals_maps}
+        which_objects: ${which_objects}
+        grasped_dtc_max_value: ${grasped_dtc_max_value}
+        env_dtc_max_value: ${env_dtc_max_value}
+        grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+        env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+        clamp_dtc: ${clamp_dtc}
+        max_contact_prob: ${max_contact_prob}
+        mask_normals_within_sdf: ${mask_normals_within_sdf}
+        dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+        adaptive_normals_mask: ${adaptive_normals_mask}
+        max_depth: ${max_depth}
+        image_shape: ${agent.desired_image_shape}
+        learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: ${contact_model_name}
+        zero_centered: false
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 15000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 250
+validate_diffusion_on_action_loss_every_epochs: 250
+train_eval_diffusion_on_action_loss_every_epochs: 250
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: 250
+save_snapshot_every_epochs_diffusion: 1500
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1