Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,18 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/1_eval_1_c21bfe5dd150e694f7bd.mp4 filter=lfs diff=lfs merge=lfs -text
+221701/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/0_eval_0_886132ad98d0c31f9311.mp4 filter=lfs diff=lfs merge=lfs -text
+221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/3_eval_3_91b538bdbc0a7862dd54.mp4 filter=lfs diff=lfs merge=lfs -text
+221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/2_eval_2_312f8f530abaf399946b.mp4 filter=lfs diff=lfs merge=lfs -text
+221701/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/4_eval_4_fdb67280e4956e7c7ab4.mp4 filter=lfs diff=lfs merge=lfs -text
+221701/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+221701/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+221701/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+221701/episode_rosbags/episode_1_2024-12-17-22-18-07.bag filter=lfs diff=lfs merge=lfs -text
+221701/episode_rosbags/episode_3_2024-12-17-22-19-40.bag filter=lfs diff=lfs merge=lfs -text
+221701/episode_rosbags/episode_2_2024-12-17-22-18-55.bag filter=lfs diff=lfs merge=lfs -text
+221701/episode_rosbags/episode_0_2024-12-17-22-17-28.bag filter=lfs diff=lfs merge=lfs -text
+221701/episode_rosbags/episode_4_2024-12-17-22-20-30.bag filter=lfs diff=lfs merge=lfs -text

221700/.hydra/config.yaml

@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '511_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${now:%H%M%S}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

221700/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_policy
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221700
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

221700/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion

221700/eval_policy.log

@@ -0,0 +1,15 @@
+[2024-12-17 22:17:00,365][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-17 22:17:00,368][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2024-12-17 22:17:03,038][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+

221701/.hydra/config.yaml

@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '511_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${now:%H%M%S}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

221701/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

221701/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion

221701/episode_rosbags/aligned_depth_to_color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

221701/episode_rosbags/cam_tf_world.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256

221701/episode_rosbags/color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

221701/episode_rosbags/depth_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200

221701/episode_rosbags/episode_0_2024-12-17-22-17-28.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef8953ac148986b87c2d4b214a4613f4fcb672077e3a0397cbfe955f6d3e864e
+size 1331933908

221701/episode_rosbags/episode_1_2024-12-17-22-18-07.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:973c5409c453f2f75649012234ac21fc1051e9fbc6f4dc7975d57a17b903bacf
+size 1333116690

221701/episode_rosbags/episode_2_2024-12-17-22-18-55.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fbfbbe4082ddeaf93eaf7e56815f20ce0e2a82facddbe125a7a6239247e7449
+size 1337452083

221701/episode_rosbags/episode_3_2024-12-17-22-19-40.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:360bfef2d2a69f4ea94560472960cd8d9154d78749622e3b49e488ed763e0c12
+size 1331313864

221701/episode_rosbags/episode_4_2024-12-17-22-20-30.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8d63b5b2a8948ea1883d8319f185a2c256ac3ec6fa690320587357916d6e3ff
+size 1332080944

221701/eval_robot.log

@@ -0,0 +1,12 @@
+[2024-12-17 22:17:01,734][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-17 22:17:01,737][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+

221701/eval_video/0_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:886132ad98d0c31f9311d14dc2c5dee96950b1fe323b0a6a400944af3ea141eb
+size 1184434

221701/eval_video/1_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c21bfe5dd150e694f7bd94f2e139debe647342304be95bebc39275ea751bb871
+size 1135643

221701/eval_video/2_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:312f8f530abaf399946b180b4b5ce328adf569df1b7b33eaa17f0cfeedd8eb91
+size 1245998

221701/eval_video/3_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91b538bdbc0a7862dd5409e3da19096cc22293420c230d1c32a5d9068e4ad636
+size 1283821

221701/eval_video/4_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdb67280e4956e7c7ab41f1c04fc31828780e6e74dad0aec932ba4621892bbb8
+size 1291419

221701/tb/events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4e5ff7b855161bca4e631495856efcf4706c51d2c3674688926e37d925f9c38
+size 1103

221701/wandb/debug-internal.log

@@ -0,0 +1,681 @@
+2024-12-17 22:17:07,590 INFO    StreamThr :84579 [internal.py:wandb_internal():85] W&B internal server running at pid: 84579, started at: 2024-12-17 22:17:07.589593
+2024-12-17 22:17:07,591 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status
+2024-12-17 22:17:07,601 INFO    WriterThread:84579 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/run-sqbcnf7o.wandb
+2024-12-17 22:17:07,601 DEBUG   SenderThread:84579 [sender.py:send():379] send: header
+2024-12-17 22:17:07,608 DEBUG   SenderThread:84579 [sender.py:send():379] send: run
+2024-12-17 22:17:07,901 INFO    SenderThread:84579 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files
+2024-12-17 22:17:07,901 INFO    SenderThread:84579 [sender.py:_start_run_threads():1188] run started: sqbcnf7o with start time 1734491827.589933
+2024-12-17 22:17:07,920 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: check_version
+2024-12-17 22:17:07,920 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: check_version
+2024-12-17 22:17:07,967 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: run_start
+2024-12-17 22:17:07,972 DEBUG   HandlerThread:84579 [system_info.py:__init__():26] System info init
+2024-12-17 22:17:07,972 DEBUG   HandlerThread:84579 [system_info.py:__init__():41] System info init done
+2024-12-17 22:17:07,972 INFO    HandlerThread:84579 [system_monitor.py:start():194] Starting system monitor
+2024-12-17 22:17:07,972 INFO    SystemMonitor:84579 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-17 22:17:07,972 INFO    HandlerThread:84579 [system_monitor.py:probe():214] Collecting system info
+2024-12-17 22:17:07,973 INFO    SystemMonitor:84579 [interfaces.py:start():188] Started cpu monitoring
+2024-12-17 22:17:07,974 INFO    SystemMonitor:84579 [interfaces.py:start():188] Started disk monitoring
+2024-12-17 22:17:07,974 INFO    SystemMonitor:84579 [interfaces.py:start():188] Started gpu monitoring
+2024-12-17 22:17:07,974 INFO    SystemMonitor:84579 [interfaces.py:start():188] Started memory monitoring
+2024-12-17 22:17:07,975 INFO    SystemMonitor:84579 [interfaces.py:start():188] Started network monitoring
+2024-12-17 22:17:07,984 DEBUG   HandlerThread:84579 [system_info.py:probe():152] Probing system
+2024-12-17 22:17:07,986 DEBUG   HandlerThread:84579 [system_info.py:_probe_git():137] Probing git
+2024-12-17 22:17:07,991 DEBUG   HandlerThread:84579 [system_info.py:_probe_git():145] Probing git done
+2024-12-17 22:17:07,991 DEBUG   HandlerThread:84579 [system_info.py:probe():200] Probing system done
+2024-12-17 22:17:07,991 DEBUG   HandlerThread:84579 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-18T03:17:07.984722', 'startedAt': '2024-12-18T03:17:07.579033', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '5f28b54373d7d41e4ebace380ff1080a8ffc4051'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3744.56375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.353, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.033, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 416.7548141479492}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2024-12-17 22:17:07,991 INFO    HandlerThread:84579 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-17 22:17:07,991 INFO    HandlerThread:84579 [system_monitor.py:probe():227] Publishing system info
+2024-12-17 22:17:07,991 DEBUG   HandlerThread:84579 [system_info.py:_save_code():44] Saving code
+2024-12-17 22:17:07,993 DEBUG   HandlerThread:84579 [system_info.py:_save_code():67] Saving code done
+2024-12-17 22:17:07,993 DEBUG   HandlerThread:84579 [system_info.py:_save_patches():84] Saving git patches
+2024-12-17 22:17:08,065 DEBUG   HandlerThread:84579 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-17 22:17:08,065 INFO    HandlerThread:84579 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-17 22:17:08,068 DEBUG   SenderThread:84579 [sender.py:send():379] send: files
+2024-12-17 22:17:08,068 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-17 22:17:08,069 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-17 22:17:08,069 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-17 22:17:08,301 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-17 22:17:08,301 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:17:08,301 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: python_packages
+2024-12-17 22:17:08,302 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:17:08,303 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:08,313 INFO    wandb-upload_0:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/eglwkdot-wandb-metadata.json
+2024-12-17 22:17:08,323 INFO    wandb-upload_2:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/7k7jg9di-diff.patch
+2024-12-17 22:17:08,349 INFO    HandlerThread:84579 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/tb"
+  save: true
+}
+control {
+  relay_id: "81d3cf2cbfed43cbb7d230993cc9c89e"
+}
+_info {
+  stream_id: "sqbcnf7o"
+}
+
+2024-12-17 22:17:08,349 DEBUG   HandlerThread:84579 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-17 22:17:08,395 INFO    wandb-upload_1:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/ggysc832-code/FISH/eval_robot.py
+2024-12-17 22:17:08,456 DEBUG   SenderThread:84579 [sender.py:send():379] send: telemetry
+2024-12-17 22:17:08,456 DEBUG   SenderThread:84579 [sender.py:send():379] send: tbrecord
+2024-12-17 22:17:08,456 DEBUG   SenderThread:84579 [sender.py:send():379] send: files
+2024-12-17 22:17:08,457 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0 with policy live
+2024-12-17 22:17:08,631 INFO    wandb-upload_0:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/hghsffpo-events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0
+2024-12-17 22:17:08,902 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0
+2024-12-17 22:17:08,903 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/diff.patch
+2024-12-17 22:17:08,903 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/code/FISH/eval_robot.py
+2024-12-17 22:17:08,903 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-metadata.json
+2024-12-17 22:17:08,903 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:08,903 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/requirements.txt
+2024-12-17 22:17:08,903 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/code/FISH
+2024-12-17 22:17:08,903 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/code
+2024-12-17 22:17:09,302 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:10,302 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:10,902 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:11,302 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:12,302 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:13,302 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:17:13,302 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:13,845 DEBUG   SenderThread:84579 [sender.py:send():379] send: config
+2024-12-17 22:17:14,303 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:14,903 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:15,303 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:16,303 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:17,303 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:18,302 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:17:18,303 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:18,904 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:19,303 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:20,304 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:20,904 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:21,304 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:22,304 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:23,301 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:17:23,301 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:17:23,304 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:23,434 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:17:24,304 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:24,906 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:25,304 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:26,304 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:26,906 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:27,304 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:28,305 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:28,457 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:17:29,305 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:30,305 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:30,907 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:31,305 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:32,306 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:32,908 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:33,306 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:34,303 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:17:34,306 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:35,306 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:36,306 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:36,909 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:37,306 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:38,301 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:17:38,302 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:17:38,345 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:39,307 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:40,117 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:17:40,306 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:40,910 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:40,910 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/config.yaml
+2024-12-17 22:17:41,307 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:42,307 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:43,307 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:44,307 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:44,911 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:45,303 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:17:45,307 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:46,307 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:47,307 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:48,307 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:49,308 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:50,308 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:51,113 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:17:51,308 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:52,310 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:52,913 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:53,301 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:17:53,302 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:17:53,345 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:54,310 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:55,329 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:56,211 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:17:56,225 DEBUG   SenderThread:84579 [sender.py:send():379] send: files
+2024-12-17 22:17:56,225 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_886132ad98d0c31f9311.mp4 with policy now
+2024-12-17 22:17:56,225 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 22:17:56,226 DEBUG   SenderThread:84579 [sender.py:send():379] send: history
+2024-12-17 22:17:56,226 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:17:56,228 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:17:56,311 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:56,485 INFO    wandb-upload_1:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/uq7df1o8-media/videos/eval/0_eval_0_886132ad98d0c31f9311.mp4
+2024-12-17 22:17:56,915 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:56,915 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json
+2024-12-17 22:17:56,915 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/0_eval_0_886132ad98d0c31f9311.mp4
+2024-12-17 22:17:56,915 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media
+2024-12-17 22:17:56,915 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval
+2024-12-17 22:17:56,915 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos
+2024-12-17 22:17:57,186 INFO    wandb-upload_0:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/ucxh4z2j-media/videos/eval/0_eval_0_886132ad98d0c31f9311.mp4
+2024-12-17 22:17:57,311 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:58,312 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:58,915 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:59,311 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:00,311 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:00,916 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:18:01,311 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:01,477 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:18:02,311 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:03,311 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:04,311 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:04,917 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0
+2024-12-17 22:18:04,918 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:18:05,312 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:06,312 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:06,918 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:18:07,305 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:18:07,312 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:07,975 DEBUG   SystemMonitor:84579 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-17 22:18:07,976 DEBUG   SenderThread:84579 [sender.py:send():379] send: stats
+2024-12-17 22:18:08,302 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:18:08,302 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:18:08,312 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:08,919 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:18:09,312 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:10,312 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:10,920 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:18:11,312 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:12,312 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:12,370 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:18:13,312 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:14,313 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:15,313 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:16,313 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:17,313 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:18,305 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:18:18,313 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:19,313 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:20,313 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:21,313 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:22,313 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:23,302 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:18:23,302 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:18:23,345 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:23,435 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:18:24,314 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:25,314 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:26,314 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:27,314 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:28,314 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:29,305 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:18:29,314 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:30,315 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:31,315 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:32,315 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:33,315 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:34,315 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:34,717 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:18:35,315 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:35,940 DEBUG   SenderThread:84579 [sender.py:send():379] send: files
+2024-12-17 22:18:35,940 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_c21bfe5dd150e694f7bd.mp4 with policy now
+2024-12-17 22:18:35,942 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 22:18:35,943 DEBUG   SenderThread:84579 [sender.py:send():379] send: history
+2024-12-17 22:18:35,943 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:18:35,944 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:18:36,188 INFO    wandb-upload_1:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/4cgqz4qo-media/videos/eval/1_eval_1_c21bfe5dd150e694f7bd.mp4
+2024-12-17 22:18:36,319 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:36,928 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:18:36,928 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json
+2024-12-17 22:18:36,928 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/1_eval_1_c21bfe5dd150e694f7bd.mp4
+2024-12-17 22:18:36,928 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval
+2024-12-17 22:18:37,214 INFO    wandb-upload_0:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/9sjtqb51-media/videos/eval/1_eval_1_c21bfe5dd150e694f7bd.mp4
+2024-12-17 22:18:37,318 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:37,976 DEBUG   SenderThread:84579 [sender.py:send():379] send: stats
+2024-12-17 22:18:38,306 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:18:38,307 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:18:38,349 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:39,332 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:39,791 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:18:40,335 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:40,929 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:18:41,338 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:42,335 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:43,336 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:44,339 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:44,930 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:18:45,309 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:18:45,337 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:46,336 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:47,335 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:48,348 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:49,350 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:50,328 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:18:50,340 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:51,343 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:52,352 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:52,739 DEBUG   SenderThread:84579 [sender.py:send():379] send: history
+2024-12-17 22:18:52,739 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:18:52,740 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:18:52,933 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0
+2024-12-17 22:18:52,933 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json
+2024-12-17 22:18:53,180 INFO    wandb-upload_1:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/qz8moi18-events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0
+2024-12-17 22:18:53,305 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:18:53,305 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:18:53,340 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:54,353 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:54,934 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:18:55,356 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:55,397 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:18:56,352 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:56,934 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:18:57,348 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:58,348 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:58,935 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:18:59,348 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:00,349 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:01,311 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:19:01,349 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:02,348 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:03,348 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:04,350 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:05,349 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:06,312 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:19:06,350 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:07,351 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:07,976 DEBUG   SenderThread:84579 [sender.py:send():379] send: stats
+2024-12-17 22:19:08,306 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:19:08,307 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:19:08,350 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:09,350 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:10,351 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:11,351 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:11,381 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:19:12,352 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:13,352 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:14,352 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:15,352 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:16,353 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:17,313 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:19:17,353 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:18,353 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:19,353 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:20,354 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:21,354 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:22,357 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:23,310 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:19:23,310 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:19:23,357 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:23,385 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:19:24,355 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:25,368 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:25,684 DEBUG   SenderThread:84579 [sender.py:send():379] send: files
+2024-12-17 22:19:25,684 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_312f8f530abaf399946b.mp4 with policy now
+2024-12-17 22:19:25,689 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 22:19:25,690 DEBUG   SenderThread:84579 [sender.py:send():379] send: history
+2024-12-17 22:19:25,690 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:19:25,691 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:19:25,933 INFO    wandb-upload_0:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/qyjuq5lx-media/videos/eval/2_eval_2_312f8f530abaf399946b.mp4
+2024-12-17 22:19:25,944 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json
+2024-12-17 22:19:25,944 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/2_eval_2_312f8f530abaf399946b.mp4
+2024-12-17 22:19:25,944 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval
+2024-12-17 22:19:26,186 INFO    wandb-upload_1:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/e6gxg7js-media/videos/eval/2_eval_2_312f8f530abaf399946b.mp4
+2024-12-17 22:19:26,367 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:26,944 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:19:27,375 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:28,377 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:29,316 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:19:29,382 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:30,379 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:30,946 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:19:31,379 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:32,413 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:33,390 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:34,389 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:34,947 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:19:35,316 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:19:35,391 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:36,398 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:36,589 DEBUG   SenderThread:84579 [sender.py:send():379] send: history
+2024-12-17 22:19:36,589 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:19:36,590 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:19:36,948 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0
+2024-12-17 22:19:36,948 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json
+2024-12-17 22:19:37,394 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:37,977 DEBUG   SenderThread:84579 [sender.py:send():379] send: stats
+2024-12-17 22:19:38,351 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:19:38,352 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:19:38,401 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:38,948 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:19:39,407 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:40,403 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:40,949 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:19:41,045 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:19:41,408 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:42,399 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:43,416 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:44,399 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:44,950 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:19:45,399 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:46,399 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:46,505 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:19:47,399 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:48,399 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:49,399 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:50,399 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:51,400 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:52,316 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:19:52,400 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:53,323 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:19:53,323 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:19:53,400 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:54,400 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:55,400 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:56,400 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:57,400 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:57,427 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:19:58,400 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:59,400 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:00,400 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:01,400 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:02,401 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:03,316 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:20:03,401 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:04,404 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:05,402 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:06,402 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:07,402 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:07,977 DEBUG   SenderThread:84579 [sender.py:send():379] send: stats
+2024-12-17 22:20:08,316 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:20:08,323 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:20:08,323 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:20:08,402 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:09,403 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:10,403 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:10,959 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:20:11,130 DEBUG   SenderThread:84579 [sender.py:send():379] send: files
+2024-12-17 22:20:11,130 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_91b538bdbc0a7862dd54.mp4 with policy now
+2024-12-17 22:20:11,135 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 22:20:11,135 DEBUG   SenderThread:84579 [sender.py:send():379] send: history
+2024-12-17 22:20:11,135 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:20:11,136 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:20:11,398 INFO    wandb-upload_0:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/6c53q71e-media/videos/eval/3_eval_3_91b538bdbc0a7862dd54.mp4
+2024-12-17 22:20:11,416 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:11,959 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json
+2024-12-17 22:20:11,959 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/3_eval_3_91b538bdbc0a7862dd54.mp4
+2024-12-17 22:20:11,960 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval
+2024-12-17 22:20:12,225 INFO    wandb-upload_1:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/g46ird7n-media/videos/eval/3_eval_3_91b538bdbc0a7862dd54.mp4
+2024-12-17 22:20:12,408 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:12,960 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:20:13,327 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:20:13,415 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:14,427 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:15,426 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:16,424 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:16,961 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:20:17,424 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:18,428 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:18,962 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:20:19,322 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:20:19,464 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:20,433 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:21,435 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:22,435 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:23,328 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:20:23,329 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:20:23,444 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:24,406 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:20:24,450 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:25,449 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:26,461 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:27,464 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:27,673 DEBUG   SenderThread:84579 [sender.py:send():379] send: history
+2024-12-17 22:20:27,673 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:20:27,673 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:20:27,965 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0
+2024-12-17 22:20:27,965 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json
+2024-12-17 22:20:28,200 INFO    wandb-upload_0:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/hjkbjwos-events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0
+2024-12-17 22:20:28,457 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:28,965 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:20:29,457 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:30,336 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:20:30,457 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:30,966 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:20:31,457 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:32,459 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:32,966 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:20:33,467 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:34,457 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:34,967 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:20:35,457 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:36,162 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:20:36,457 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:36,968 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:20:37,458 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:37,978 DEBUG   SenderThread:84579 [sender.py:send():379] send: stats
+2024-12-17 22:20:38,328 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:20:38,328 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:20:38,458 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:39,459 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:40,459 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:41,459 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:41,472 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:20:42,459 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:43,459 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:44,459 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:45,460 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:46,459 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:47,333 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:20:47,459 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:48,460 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:49,461 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:50,461 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:51,461 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:52,333 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:20:52,461 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:53,331 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:20:53,331 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:20:53,463 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:54,463 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:55,463 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:56,462 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:57,470 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:57,505 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:20:58,463 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:58,975 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:20:59,463 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:00,464 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:01,463 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:02,493 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:02,529 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:21:02,795 DEBUG   SenderThread:84579 [sender.py:send():379] send: files
+2024-12-17 22:21:02,795 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_fdb67280e4956e7c7ab4.mp4 with policy now
+2024-12-17 22:21:02,798 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 22:21:02,798 DEBUG   SenderThread:84579 [sender.py:send():379] send: history
+2024-12-17 22:21:02,799 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:21:02,799 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:21:02,976 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:21:02,976 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json
+2024-12-17 22:21:02,977 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/4_eval_4_fdb67280e4956e7c7ab4.mp4
+2024-12-17 22:21:02,977 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval
+2024-12-17 22:21:03,064 INFO    wandb-upload_1:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/jpil4fgf-media/videos/eval/4_eval_4_fdb67280e4956e7c7ab4.mp4
+2024-12-17 22:21:03,333 INFO    wandb-upload_0:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/qluoymh3-media/videos/eval/4_eval_4_fdb67280e4956e7c7ab4.mp4
+2024-12-17 22:21:03,488 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:04,481 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:04,977 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:21:05,487 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:06,490 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:07,482 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:07,701 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:21:07,978 DEBUG   SenderThread:84579 [sender.py:send():379] send: stats
+2024-12-17 22:21:08,354 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:21:08,354 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:21:08,483 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:08,979 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:21:09,481 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:10,489 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:10,980 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:21:11,453 DEBUG   SenderThread:84579 [sender.py:send():379] send: exit
+2024-12-17 22:21:11,454 INFO    SenderThread:84579 [sender.py:send_exit():586] handling exit code: 0
+2024-12-17 22:21:11,454 INFO    SenderThread:84579 [sender.py:send_exit():588] handling runtime: 243
+2024-12-17 22:21:11,454 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:21:11,455 INFO    SenderThread:84579 [sender.py:send_exit():594] send defer
+2024-12-17 22:21:11,455 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:11,455 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-17 22:21:11,455 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:11,455 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-17 22:21:11,455 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 1
+2024-12-17 22:21:11,455 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:11,455 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-17 22:21:11,456 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:11,456 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-17 22:21:11,456 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 2
+2024-12-17 22:21:11,456 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:11,456 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-17 22:21:11,456 INFO    HandlerThread:84579 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-17 22:21:11,456 DEBUG   SystemMonitor:84579 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-17 22:21:11,456 INFO    HandlerThread:84579 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-17 22:21:11,456 DEBUG   SystemMonitor:84579 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-17 22:21:11,456 INFO    HandlerThread:84579 [interfaces.py:finish():200] Joined disk monitor
+2024-12-17 22:21:11,502 INFO    HandlerThread:84579 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-17 22:21:11,502 INFO    HandlerThread:84579 [interfaces.py:finish():200] Joined memory monitor
+2024-12-17 22:21:11,502 INFO    HandlerThread:84579 [interfaces.py:finish():200] Joined network monitor
+2024-12-17 22:21:11,503 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:11,503 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-17 22:21:11,503 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 3
+2024-12-17 22:21:11,503 DEBUG   SenderThread:84579 [sender.py:send():379] send: stats
+2024-12-17 22:21:11,503 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:11,503 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-17 22:21:11,503 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:11,503 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-17 22:21:11,503 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 4
+2024-12-17 22:21:11,503 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:11,503 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-17 22:21:11,980 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0
+2024-12-17 22:21:11,980 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json
+2024-12-17 22:21:12,981 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:21:17,505 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:21:17,505 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:21:17,505 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:17,506 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-17 22:21:17,506 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-17 22:21:17,506 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 5
+2024-12-17 22:21:17,506 DEBUG   SenderThread:84579 [sender.py:send():379] send: history
+2024-12-17 22:21:17,506 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:17,506 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:21:17,506 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-17 22:21:17,507 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:21:17,507 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:21:17,507 DEBUG   SenderThread:84579 [sender.py:send():379] send: history
+2024-12-17 22:21:17,507 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:21:17,511 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:21:17,511 DEBUG   SenderThread:84579 [sender.py:send():379] send: summary
+2024-12-17 22:21:17,511 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:21:17,511 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:17,512 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-17 22:21:17,512 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 6
+2024-12-17 22:21:17,512 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:17,512 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-17 22:21:17,512 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:17,512 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-17 22:21:17,512 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 7
+2024-12-17 22:21:17,512 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:21:17,512 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:17,512 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-17 22:21:17,512 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:17,513 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-17 22:21:17,982 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json
+2024-12-17 22:21:18,454 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:21:18,708 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 8
+2024-12-17 22:21:18,708 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:21:18,708 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:18,709 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-17 22:21:18,709 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:18,709 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-17 22:21:18,709 INFO    SenderThread:84579 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-17 22:21:18,715 INFO    SenderThread:84579 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-17 22:21:18,731 INFO    SenderThread:84579 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-17 22:21:18,743 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 9
+2024-12-17 22:21:18,743 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:18,743 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-17 22:21:18,743 DEBUG   SenderThread:84579 [sender.py:send():379] send: artifact
+2024-12-17 22:21:18,983 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:21:19,455 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:21:19,527 INFO    wandb-upload_2:84579 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpqschr5qi
+2024-12-17 22:21:19,527 INFO    wandb-upload_0:84579 [upload_job.py:push():85] Skipped uploading /tmp/tmpsbqi9vuh/wandb-job.json
+2024-12-17 22:21:19,649 INFO    wandb-upload_1:84579 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpfksjk6nr
+2024-12-17 22:21:20,210 INFO    SenderThread:84579 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM4MzQxMDY4OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM4MzM4ODU2OA==', 'versionIndex': 383}}}
+2024-12-17 22:21:20,210 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:20,210 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-17 22:21:20,210 INFO    SenderThread:84579 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-17 22:21:20,984 INFO    SenderThread:84579 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files
+2024-12-17 22:21:20,984 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/config.yaml config.yaml
+2024-12-17 22:21:20,984 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json wandb-summary.json
+2024-12-17 22:21:20,984 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/requirements.txt requirements.txt
+2024-12-17 22:21:20,984 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0 events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0
+2024-12-17 22:21:20,987 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-metadata.json wandb-metadata.json
+2024-12-17 22:21:20,987 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log output.log
+2024-12-17 22:21:20,987 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/diff.patch diff.patch
+2024-12-17 22:21:20,989 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-17 22:21:20,989 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/2_eval_2_312f8f530abaf399946b.mp4 media/videos/eval/2_eval_2_312f8f530abaf399946b.mp4
+2024-12-17 22:21:20,989 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/0_eval_0_886132ad98d0c31f9311.mp4 media/videos/eval/0_eval_0_886132ad98d0c31f9311.mp4
+2024-12-17 22:21:20,989 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/3_eval_3_91b538bdbc0a7862dd54.mp4 media/videos/eval/3_eval_3_91b538bdbc0a7862dd54.mp4
+2024-12-17 22:21:20,989 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/4_eval_4_fdb67280e4956e7c7ab4.mp4 media/videos/eval/4_eval_4_fdb67280e4956e7c7ab4.mp4
+2024-12-17 22:21:20,989 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/1_eval_1_c21bfe5dd150e694f7bd.mp4 media/videos/eval/1_eval_1_c21bfe5dd150e694f7bd.mp4
+2024-12-17 22:21:20,989 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 10
+2024-12-17 22:21:20,989 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:21:20,989 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:20,990 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-17 22:21:20,992 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:20,992 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-17 22:21:20,992 INFO    SenderThread:84579 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 22:21:21,167 INFO    wandb-upload_0:84579 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json
+2024-12-17 22:21:21,200 INFO    wandb-upload_1:84579 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/requirements.txt
+2024-12-17 22:21:21,217 INFO    wandb-upload_2:84579 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/config.yaml
+2024-12-17 22:21:21,265 INFO    wandb-upload_3:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/r373gfgp-events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0
+2024-12-17 22:21:21,320 INFO    wandb-upload_4:84579 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:21:21,456 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:21:21,456 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:21:21,521 INFO    Thread-11 (_thread_body):84579 [sender.py:transition_state():614] send defer: 11
+2024-12-17 22:21:21,521 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:21,521 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-17 22:21:21,521 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:21,521 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-17 22:21:21,521 INFO    SenderThread:84579 [file_pusher.py:join():175] waiting for file pusher
+2024-12-17 22:21:21,522 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 12
+2024-12-17 22:21:21,522 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:21,522 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-17 22:21:21,522 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:21,523 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-17 22:21:21,523 INFO    SenderThread:84579 [file_stream.py:finish():601] file stream finish called
+2024-12-17 22:21:21,596 INFO    SenderThread:84579 [file_stream.py:finish():605] file stream finish is done
+2024-12-17 22:21:21,596 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 13
+2024-12-17 22:21:21,597 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:21,597 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-17 22:21:21,597 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:21,597 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-17 22:21:21,597 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 14
+2024-12-17 22:21:21,597 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:21,597 DEBUG   SenderThread:84579 [sender.py:send():379] send: final
+2024-12-17 22:21:21,597 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-17 22:21:21,597 DEBUG   SenderThread:84579 [sender.py:send():379] send: footer
+2024-12-17 22:21:21,597 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:21,598 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-17 22:21:21,598 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:21:21,598 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:21:21,598 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: server_info
+2024-12-17 22:21:21,598 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: server_info
+2024-12-17 22:21:21,600 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:21:21,600 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-17 22:21:21,600 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-17 22:21:21,600 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:21,661 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:21:21,662 INFO    MainThread:84579 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-17 22:21:21,662 INFO    MainThread:84579 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-17 22:21:21,662 INFO    MainThread:84579 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-17 22:21:21,662 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-17 22:21:21,662 INFO    HandlerThread:84579 [handler.py:finish():882] shutting down handler
+2024-12-17 22:21:22,600 INFO    WriterThread:84579 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/run-sqbcnf7o.wandb
+2024-12-17 22:21:22,662 INFO    SenderThread:84579 [sender.py:finish():1615] shutting down sender
+2024-12-17 22:21:22,662 INFO    SenderThread:84579 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 22:21:22,662 INFO    SenderThread:84579 [file_pusher.py:join():175] waiting for file pusher

221701/wandb/debug.log

@@ -0,0 +1,31 @@
+2024-12-17 22:17:07,582 INFO    MainThread:84391 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-17 22:17:07,582 INFO    MainThread:84391 [wandb_setup.py:_flush():76] Configure stats pid to 84391
+2024-12-17 22:17:07,582 INFO    MainThread:84391 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-17 22:17:07,582 INFO    MainThread:84391 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/settings
+2024-12-17 22:17:07,582 INFO    MainThread:84391 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-17 22:17:07,583 INFO    MainThread:84391 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-17 22:17:07,583 INFO    MainThread:84391 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-17 22:17:07,583 INFO    MainThread:84391 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-17 22:17:07,583 INFO    MainThread:84391 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/logs/debug.log
+2024-12-17 22:17:07,583 INFO    MainThread:84391 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/logs/debug-internal.log
+2024-12-17 22:17:07,583 INFO    MainThread:84391 [wandb_init.py:init():569] calling init triggers
+2024-12-17 22:17:07,583 INFO    MainThread:84391 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '511_0', 'wandb_notes': '511_0_', 'eval': True, 'true_action_history': False, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': False, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['electrodyn_twodim'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/511_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701'}
+2024-12-17 22:17:07,584 INFO    MainThread:84391 [wandb_init.py:init():619] starting backend
+2024-12-17 22:17:07,584 INFO    MainThread:84391 [wandb_init.py:init():623] setting up manager
+2024-12-17 22:17:07,588 INFO    MainThread:84391 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-17 22:17:07,589 INFO    MainThread:84391 [wandb_init.py:init():631] backend started and connected
+2024-12-17 22:17:07,600 INFO    MainThread:84391 [wandb_init.py:init():720] updated telemetry
+2024-12-17 22:17:07,606 INFO    MainThread:84391 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-17 22:17:07,920 INFO    MainThread:84391 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-17 22:17:07,962 INFO    MainThread:84391 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-17 22:17:07,962 INFO    MainThread:84391 [wandb_init.py:init():804] starting run threads in backend
+2024-12-17 22:17:08,301 INFO    MainThread:84391 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-17 22:17:08,301 INFO    MainThread:84391 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-17 22:17:08,301 INFO    MainThread:84391 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-17 22:17:08,303 INFO    MainThread:84391 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-17 22:17:08,305 INFO    MainThread:84391 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-17 22:17:08,305 INFO    MainThread:84391 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/tb, True
+2024-12-17 22:17:13,844 INFO    MainThread:84391 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'electrodyn', 'left_book_slot': 'twodim'}
+2024-12-17 22:21:22,664 WARNING MsgRouterThr:84391 [router.py:message_loop():77] message_loop has been closed

221701/wandb/run-20241217_221707-sqbcnf7o/files/code/FISH/eval_robot.py

@@ -0,0 +1,512 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)		
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()

221701/wandb/run-20241217_221707-sqbcnf7o/files/config.yaml

@@ -0,0 +1,891 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 0
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '511_0'
+wandb_notes:
+  desc: null
+  value: '511_0_'
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: false
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: false
+use_sdf_maps:
+  desc: null
+  value: false
+use_normals_maps:
+  desc: null
+  value: false
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value:
+  - electrodyn_twodim
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 0
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/kostas-graid/datasets/extrinsic_contact_data/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/511_0
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: false
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: false
+        use_sdf_maps: false
+        use_normals_maps: false
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 0
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: false
+          use_sdf_maps: false
+          use_normals_maps: false
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1734491827
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 16
+      - 23
+      - 35
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: electrodyn
+left_book_slot:
+  desc: null
+  value: twodim

221701/wandb/run-20241217_221707-sqbcnf7o/files/diff.patch

@@ -0,0 +1,195 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index bb0a413..e3ce3e8 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -107,7 +107,7 @@ num_valid_demos: null
+ val_num_groups: 3
+ 
+ # name_of_expert_demo: 'book_insertion_wo_ends_wedge_v2_76_all_demos_10hz_no_filter_wo_flow_mask_expert_demos_imp_act'
+-name_of_expert_demo: '192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act'
++name_of_expert_demo: '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act'
+ expert_dataset_dirpath: '${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}'
+ 
+ expert_dataset: '${expert_dataset_dirpath}/demos.zarr'
+@@ -123,89 +123,28 @@ bc_weight_type: 'qfilter' # linear, qfilter
+ 
+ # Load weights
+ load_checkpoint: ${agent.load_checkpoint}
+-# 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# full 32 demos
+-# wandb_run_id: "265625" # RGBD history seed 0
+-# wandb_run_id: "266370_1" # RGBD history seed 1
+-# wandb_run_id: "266370_2" # RGBD history seed 2
+-# wandb_run_id: "266382_4" # RGBD history seed 4
+-
+-# 16 held out demos (16 demos)
+-# wandb_run_id: "266392_0" # RGBD+act history dataset seed 0
+-# wandb_run_id: "266407_0" # RGBD+act history dataset seed 1
+-wandb_run_id: "266406_0" # RGBD+act history dataset seed 2
+-
+-# wandb_run_id: "265626" # RGBD+Mask+act history seed 0
+-# wandb_run_id: "265638" # RGBD+Mask+act history seed 1
+-# wandb_run_id: "265637" # RGBD+Mask+act history seed 2
+-# wandb_run_id: "265636" # RGBD+Mask+act history seed 3
+-# wandb_run_id: "265635" # RGBD+Mask+act history seed 4
+-
+-# wandb_run_id: "265628" # RGBD+Mask+contact+act history seed 0
+-# wandb_run_id: "265630" # RGBD+Mask+contact+act history seed 1
+-# wandb_run_id: "265631" # RGBD+Mask+contact+act history seed 2
+-# wandb_run_id: "265633" # RGBD+Mask+contact+act history seed 3
+-# wandb_run_id: "265634" # RGBD+Mask+contact+act history seed 4
+-
+-# 8 held out demos (26 demos)
+-# wandb_run_id: "265652" # RGBD+Mask+contact+act history dataset seed 0
+-# wandb_run_id: "265654" # RGBD+Mask+contact+act history dataset seed 1
+-# wandb_run_id: "265655" # RGBD+Mask+contact+act history dataset seed 2
+-
+-# 8 held out demos (26 demos)
+-# wandb_run_id: "265656" # RGBD+Mask+act history dataset seed 0
+-# wandb_run_id: "265657" # RGBD+Mask+act history dataset seed 1
+-# wandb_run_id: "265658" # RGBD+Mask+act history dataset seed 2
+-
+-# 16 held out demos (16 demos)
+-# wandb_run_id: "265641" # RGBD+Mask+act history dataset seed 0
+-# wandb_run_id: "265644" # RGBD+Mask+act history dataset seed 1
+-# wandb_run_id: "265645" # RGBD+Mask+act history dataset seed 2
+-# wandb_run_id: "266381_0" # RGBD+Mask+act history dataset seed 4
+-
+-# 16 held out demos (16 demos)
+-# wandb_run_id: "265646" # RGBD+Mask+Contact+act history dataset seed 0
+-# wandb_run_id: "265649" # RGBD+Mask+Contact+act history dataset seed 1
+-# wandb_run_id: "265650" # RGBD+Mask+Contact+act history dataset seed 2
+-# wandb_run_id: "266378_0" # RGBD+Mask+Contact+act history dataset seed 4
+-
+-
+-
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# full dataset
+-# 6 step action history
+-# wandb_run_id: '250358_0' # RGBD+act history
+-# wandb_run_id: '250371_0' # RGBD+mask+act history
+-# wandb_run_id: '250366_0' # RGBD+contact+act history
+-# wandb_run_id: '250376_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250366_0' # RGBD+contact+act history
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# 16 held out demos
+-# wandb_run_id: '250362_0' # RGBD+act history
+-# wandb_run_id: '250373_0' # RGBD+mask+act history
+-# wandb_run_id: '250378_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250368_0' # RGBD+contact+act history
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# 32 held out demos
+-# wandb_run_id: '250365_0' # RGBD+act history
+-# wandb_run_id: '250375_0' # RGBD+mask+act history
+-wandb_run_id: '250380_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250370_0' # RGBD+contact+act history
+-# 4 step action history
+-# wandb_run_id: '264772_0' # RGBD+act history
+-# wandb_run_id: '264775_0' # RGBD+mask+act history
+-# wandb_run_id: '264776_0' # RGBD+contact+act history
+-# wandb_run_id: '264777_0' # RGBD+mask+contact+act history
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# 16 held out demos
+-# wandb_run_id: '250362_0' # RGBD+act history
+-# wandb_run_id: '250373_0' # RGBD+mask+act history
+-# wandb_run_id: '250378_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250368_0' # RGBD+contact+act history
++# 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
++# 14 demos
++
++# RGBD+act history 
++# wandb_run_id: '497_0' # german
++# wandb_run_id: '505_0' # hbm
++# wandb_run_id: '506_0' # lady
++# wandb_run_id: '507_0' # greece
++# wandb_run_id: '508_0' # fowlers
++# wandb_run_id: '509_0' # lib
++# wandb_run_id: '510_0' # modelsys
++wandb_run_id: '511_0' # electrodyn
++
++# RGBD+mask+act history
++# wandb_run_id: '512_0' # german
++# wandb_run_id: '520_0' # hbm
++# wandb_run_id: '521_0' # lady
++# wandb_run_id: '522_0' # greece
++# wandb_run_id: '523_0' # fowlers
++# wandb_run_id: '524_0' # lib
++# wandb_run_id: '525_0' # modelsys
++# wandb_run_id: '526_0' # electrodyn
+ 
+ # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # 32 held out demos
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index 9406a46..0f225ec 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -15,12 +15,24 @@ if not local_checkpoint_root_dir.exists():
+     local_checkpoint_root_dir.mkdir(parents=True)
+ 
+ # Download the model checkpoints
+-remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1')
++# remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1')
++remote_checkpoint_root_dir = Path('/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1')
+ remote_checkpoint_root_dir = remote_checkpoint_root_dir.expanduser()
+ remote_username = 'leonmkim'
+-remote_host = 'halley'
++remote_host = 'grasp-login1'
+ 
+ #180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
++# run_id_list = [
++#     '512_0', # german
++#     '520_0', # hbm
++#     '521_0', # lady
++#     '522_0', # greece
++#     '523_0', # fowlers
++#     '524_0', # lib
++#     '525_0', # modelsys
++#     '526_0', # electrodyn
++# ]
++
+ # run_id_list = [
+ #     '265638',
+ #     '265637',
+@@ -75,6 +87,17 @@ remote_host = 'halley'
+ # ]
+ 
+ # 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
++# run_id_list = [
++#     '497_0', # german
++#     '505_0', # hbm
++#     '506_0', # lady
++#     '507_0', # greece
++#     '508_0', # fowlers
++#     '509_0', # lib
++#     '510_0', # modelsys
++#     '511_0', # electrodyn
++# ]
++
+ # run_id_list = [
+ #     '266382_4',
+ #     '266370_2',
+@@ -116,13 +139,13 @@ remote_host = 'halley'
+ # ]
+ 
+ # 180x240_1_RGB_D_2.0_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+-run_id_list = [
+-    # '266384_4,# not yet available
+-    '266372_1',
+-    '266372_2',
+-    '265627',
+-    '265620',
+-]
++# run_id_list = [
++#     # '266384_4,# not yet available
++#     '266372_1',
++#     '266372_2',
++#     '265627',
++#     '265620',
++# ]
+ checkpoint_epoch = 12000
+ 
+ # use subprocess to download the model checkpoints in parallel

221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/0_eval_0_886132ad98d0c31f9311.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:886132ad98d0c31f9311d14dc2c5dee96950b1fe323b0a6a400944af3ea141eb
+size 1184434

221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/1_eval_1_c21bfe5dd150e694f7bd.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c21bfe5dd150e694f7bd94f2e139debe647342304be95bebc39275ea751bb871
+size 1135643

221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/2_eval_2_312f8f530abaf399946b.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:312f8f530abaf399946b180b4b5ce328adf569df1b7b33eaa17f0cfeedd8eb91
+size 1245998

221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/3_eval_3_91b538bdbc0a7862dd54.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91b538bdbc0a7862dd5409e3da19096cc22293420c230d1c32a5d9068e4ad636
+size 1283821

221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/4_eval_4_fdb67280e4956e7c7ab4.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdb67280e4956e7c7ab41f1c04fc31828780e6e74dad0aec932ba4621892bbb8
+size 1291419

221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log

@@ -0,0 +1,223 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1734491833.846049]: resetting environment
+[INFO] [1734491833.847335]: cleared current plan
+[INFO] [1734491833.849804]: moving to home
+[INFO] [1734491836.052531]: reached home
+[INFO] [1734491836.054112]: reset action history
+[INFO] [1734491838.763563]: environment reset
+Starting episode 0
+[INFO] [1734491838.765642]: resetting environment
+[INFO] [1734491838.766500]: cleared current plan
+[INFO] [1734491838.767396]: moving to home
+[INFO] [1734491839.769288]: reached home
+[INFO] [1734491839.770739]: reset action history
+[INFO] [1734491842.478743]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1734491844.741133]: resetting environment
+[INFO] [1734491844.742833]: cleared current plan
+[INFO] [1734491844.743701]: moving to home
+[INFO] [1734491845.747870]: reached home
+[INFO] [1734491845.749872]: reset action history
+[INFO] [1734491848.457115]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/episode_rosbags/episode_0_2024-12-17-22-17-28.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-84391.log
+started roslaunch server http://158.130.50.37:45207/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [84702]
+process[print_text-2]: started with pid [84703]
+process[rosbag_record-3]: started with pid [84704]
+started bagging!
+For topic gripper_width: timestamp difference is -109711782 for nearest: 1734491850298011645 - target: 1734491850407723427 at idx 31
+For topic gripper_width: timestamp difference is -110186509 for nearest: 1734491854498027631 - target: 1734491854608214140 at idx 91
+For topic gripper_width: timestamp difference is -106731675 for nearest: 1734491858898039081 - target: 1734491859004770756 at idx 142
+For topic gripper_width: timestamp difference is -104340164 for nearest: 1734491861898012789 - target: 1734491862002352953 at idx 178
+For topic gripper_width: timestamp difference is -105522727 for nearest: 1734491869898045684 - target: 1734491870003568411 at idx 249
+For topic gripper_width: timestamp difference is -101851860 for nearest: 1734491871698060831 - target: 1734491871799912691 at idx 249
+[INFO] [1734491875.288220]: Storing episode...
+[rosbag_record-3] killing on exit
+[print_text-2] killing on exit
+[pub_text-1] killing on exit
+[INFO] [1734491876.013628]: Stored episode 1.
+[INFO] [1734491876.015687]: Saving video...
+[INFO] [1734491876.210772]: Video saved!
+Starting episode 1
+[INFO] [1734491876.226238]: resetting environment
+[INFO] [1734491876.226459]: cleared current plan
+[INFO] [1734491876.226963]: moving to home
+[INFO] [1734491879.429263]: reached home
+[INFO] [1734491879.429749]: reset action history
+[INFO] [1734491882.141263]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1734491884.260438]: resetting environment
+[INFO] [1734491884.261181]: cleared current plan
+[INFO] [1734491884.261426]: moving to home
+[INFO] [1734491885.262947]: reached home
+[INFO] [1734491885.263611]: reset action history
+[INFO] [1734491887.969921]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/episode_rosbags/episode_1_2024-12-17-22-18-07.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-84391.log
+started roslaunch server http://158.130.50.37:41307/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [84819]
+process[print_text-5]: started with pid [84820]
+process[rosbag_record-6]: started with pid [84821]
+started bagging!
+For topic gripper_width: timestamp difference is -129322082 for nearest: 1734491913898010701 - target: 1734491914027332783 at idx 249
+[INFO] [1734491914.717205]: Storing episode...
+[rosbag_record-6] killing on exit
+[pub_text-4] killing on exit
+[print_text-5] killing on exit
+[INFO] [1734491915.487627]: Stored episode 2.
+[INFO] [1734491915.487967]: Saving video...
+[INFO] [1734491915.877498]: Video saved!
+Starting episode 2
+[INFO] [1734491915.944671]: resetting environment
+[INFO] [1734491915.957308]: cleared current plan
+[INFO] [1734491915.973571]: moving to home
+[INFO] [1734491919.787966]: reached home
+[INFO] [1734491919.799530]: reset action history
+[INFO] [1734491922.641267]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode![INFO] [1734491932.086970]: resetting environment
+[INFO] [1734491932.098732]: cleared current plan
+[INFO] [1734491932.104779]: moving to home
+[INFO] [1734491933.128375]: reached home
+[INFO] [1734491933.139505]: reset action history
+[INFO] [1734491935.969273]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/episode_rosbags/episode_2_2024-12-17-22-18-55.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-84391.log
+started roslaunch server http://158.130.50.37:39349/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [84974]
+process[print_text-8]: started with pid [84975]
+process[rosbag_record-9]: started with pid [84976]
+started bagging!
+[INFO] [1734491964.518884]: Storing episode...
+[print_text-8] killing on exit
+[rosbag_record-9] killing on exit
+[pub_text-7] killing on exit
+[INFO] [1734491965.369236]: Stored episode 3.
+[INFO] [1734491965.387140]: Saving video...
+[INFO] [1734491965.630860]: Video saved!
+Starting episode 3
+[INFO] [1734491965.692554]: resetting environment
+[INFO] [1734491965.698086]: cleared current plan
+[INFO] [1734491965.699894]: moving to home
+[INFO] [1734491969.509999]: reached home
+[INFO] [1734491969.514150]: reset action history
+[INFO] [1734491972.371009]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode![INFO] [1734491976.167610]: resetting environment
+[INFO] [1734491976.182231]: cleared current plan
+[INFO] [1734491976.192721]: moving to home
+[INFO] [1734491977.196391]: reached home
+[INFO] [1734491977.198185]: reset action history
+[INFO] [1734491980.001106]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/episode_rosbags/episode_3_2024-12-17-22-19-40.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-84391.log
+started roslaunch server http://158.130.50.37:33129/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-10]: started with pid [85103]
+process[print_text-11]: started with pid [85104]
+process[rosbag_record-12]: started with pid [85105]
+started bagging!
+[INFO] [1734492010.009368]: Storing episode...
+[rosbag_record-12] killing on exit
+[print_text-11] killing on exit
+[pub_text-10] killing on exit
+[INFO] [1734492010.816927]: Stored episode 4.
+[INFO] [1734492010.828678]: Saving video...
+[INFO] [1734492011.050094]: Video saved!
+Starting episode 4
+[INFO] [1734492011.138505]: resetting environment
+[INFO] [1734492011.141005]: cleared current plan
+[INFO] [1734492011.147700]: moving to home
+[INFO] [1734492014.972568]: reached home
+[INFO] [1734492014.985195]: reset action history
+[INFO] [1734492017.778256]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode![INFO] [1734492027.125289]: resetting environment
+[INFO] [1734492027.131171]: cleared current plan
+[INFO] [1734492027.134957]: moving to home
+[INFO] [1734492028.157503]: reached home
+[INFO] [1734492028.161432]: reset action history
+[INFO] [1734492030.960144]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/episode_rosbags/episode_4_2024-12-17-22-20-30.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-84391.log
+started roslaunch server http://158.130.50.37:34115/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-13]: started with pid [85231]
+process[print_text-14]: started with pid [85232]
+process[rosbag_record-15]: started with pid [85233]
+started bagging!
+For topic gripper_width: timestamp difference is -113954444 for nearest: 1734492056298041921 - target: 1734492056411996365 at idx 249
+For topic gripper_width: timestamp difference is -110685606 for nearest: 1734492060298010092 - target: 1734492060408695698 at idx 249
+For topic gripper_width: timestamp difference is -110269352 for nearest: 1734492060897959904 - target: 1734492061008229256 at idx 249
+[INFO] [1734492061.671327]: Storing episode...
+[rosbag_record-15] killing on exit
+[print_text-14] killing on exit
+[pub_text-13] killing on exit
+[INFO] [1734492062.525259]: Stored episode 5.
+[INFO] [1734492062.533330]: Saving video...
+[INFO] [1734492062.748301]: Video saved!
+[INFO] [1734492062.799201]: resetting environment
+[INFO] [1734492062.815422]: cleared current plan
+[INFO] [1734492062.826227]: moving to home
+[INFO] [1734492066.650764]: reached home
+[INFO] [1734492066.660339]: reset action history
+[INFO] [1734492069.590726]: environment reset
+Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!

221701/wandb/run-20241217_221707-sqbcnf7o/files/requirements.txt

@@ -0,0 +1,339 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.2
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0

221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-metadata.json

@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2024-12-18T03:17:07.984722",
+    "startedAt": "2024-12-18T03:17:07.579033",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "5f28b54373d7d41e4ebace380ff1080a8ffc4051"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 3744.56375,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4496.353,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4500.033,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 416.7548141479492
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}

221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json

@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "886132ad98d0c31f9311d14dc2c5dee96950b1fe323b0a6a400944af3ea141eb", "size": 1184434, "path": "media/videos/eval/0_eval_0_886132ad98d0c31f9311.mp4"}, "global_step": 660, "_timestamp": 1734492071.4137464, "_runtime": 243.82381343841553, "_step": 9, "eval/1_eval": {"_type": "video-file", "sha256": "c21bfe5dd150e694f7bd94f2e139debe647342304be95bebc39275ea751bb871", "size": 1135643, "path": "media/videos/eval/1_eval_1_c21bfe5dd150e694f7bd.mp4"}, "eval/num_success": 4.0, "episode": 5.0, "eval/success_rate": 0.800000011920929, "eval/2_eval": {"_type": "video-file", "sha256": "312f8f530abaf399946b180b4b5ce328adf569df1b7b33eaa17f0cfeedd8eb91", "size": 1245998, "path": "media/videos/eval/2_eval_2_312f8f530abaf399946b.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "91b538bdbc0a7862dd5409e3da19096cc22293420c230d1c32a5d9068e4ad636", "size": 1283821, "path": "media/videos/eval/3_eval_3_91b538bdbc0a7862dd54.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "fdb67280e4956e7c7ab41f1c04fc31828780e6e74dad0aec932ba4621892bbb8", "size": 1291419, "path": "media/videos/eval/4_eval_4_fdb67280e4956e7c7ab4.mp4"}, "_wandb": {"runtime": 243}}

221701/wandb/run-20241217_221707-sqbcnf7o/logs/debug-internal.log

@@ -0,0 +1,681 @@
+2024-12-17 22:17:07,590 INFO    StreamThr :84579 [internal.py:wandb_internal():85] W&B internal server running at pid: 84579, started at: 2024-12-17 22:17:07.589593
+2024-12-17 22:17:07,591 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status
+2024-12-17 22:17:07,601 INFO    WriterThread:84579 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/run-sqbcnf7o.wandb
+2024-12-17 22:17:07,601 DEBUG   SenderThread:84579 [sender.py:send():379] send: header
+2024-12-17 22:17:07,608 DEBUG   SenderThread:84579 [sender.py:send():379] send: run
+2024-12-17 22:17:07,901 INFO    SenderThread:84579 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files
+2024-12-17 22:17:07,901 INFO    SenderThread:84579 [sender.py:_start_run_threads():1188] run started: sqbcnf7o with start time 1734491827.589933
+2024-12-17 22:17:07,920 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: check_version
+2024-12-17 22:17:07,920 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: check_version
+2024-12-17 22:17:07,967 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: run_start
+2024-12-17 22:17:07,972 DEBUG   HandlerThread:84579 [system_info.py:__init__():26] System info init
+2024-12-17 22:17:07,972 DEBUG   HandlerThread:84579 [system_info.py:__init__():41] System info init done
+2024-12-17 22:17:07,972 INFO    HandlerThread:84579 [system_monitor.py:start():194] Starting system monitor
+2024-12-17 22:17:07,972 INFO    SystemMonitor:84579 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-17 22:17:07,972 INFO    HandlerThread:84579 [system_monitor.py:probe():214] Collecting system info
+2024-12-17 22:17:07,973 INFO    SystemMonitor:84579 [interfaces.py:start():188] Started cpu monitoring
+2024-12-17 22:17:07,974 INFO    SystemMonitor:84579 [interfaces.py:start():188] Started disk monitoring
+2024-12-17 22:17:07,974 INFO    SystemMonitor:84579 [interfaces.py:start():188] Started gpu monitoring
+2024-12-17 22:17:07,974 INFO    SystemMonitor:84579 [interfaces.py:start():188] Started memory monitoring
+2024-12-17 22:17:07,975 INFO    SystemMonitor:84579 [interfaces.py:start():188] Started network monitoring
+2024-12-17 22:17:07,984 DEBUG   HandlerThread:84579 [system_info.py:probe():152] Probing system
+2024-12-17 22:17:07,986 DEBUG   HandlerThread:84579 [system_info.py:_probe_git():137] Probing git
+2024-12-17 22:17:07,991 DEBUG   HandlerThread:84579 [system_info.py:_probe_git():145] Probing git done
+2024-12-17 22:17:07,991 DEBUG   HandlerThread:84579 [system_info.py:probe():200] Probing system done
+2024-12-17 22:17:07,991 DEBUG   HandlerThread:84579 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-18T03:17:07.984722', 'startedAt': '2024-12-18T03:17:07.579033', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '5f28b54373d7d41e4ebace380ff1080a8ffc4051'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3744.56375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.353, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.033, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 416.7548141479492}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2024-12-17 22:17:07,991 INFO    HandlerThread:84579 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-17 22:17:07,991 INFO    HandlerThread:84579 [system_monitor.py:probe():227] Publishing system info
+2024-12-17 22:17:07,991 DEBUG   HandlerThread:84579 [system_info.py:_save_code():44] Saving code
+2024-12-17 22:17:07,993 DEBUG   HandlerThread:84579 [system_info.py:_save_code():67] Saving code done
+2024-12-17 22:17:07,993 DEBUG   HandlerThread:84579 [system_info.py:_save_patches():84] Saving git patches
+2024-12-17 22:17:08,065 DEBUG   HandlerThread:84579 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-17 22:17:08,065 INFO    HandlerThread:84579 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-17 22:17:08,068 DEBUG   SenderThread:84579 [sender.py:send():379] send: files
+2024-12-17 22:17:08,068 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-17 22:17:08,069 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-17 22:17:08,069 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-17 22:17:08,301 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-17 22:17:08,301 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:17:08,301 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: python_packages
+2024-12-17 22:17:08,302 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:17:08,303 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:08,313 INFO    wandb-upload_0:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/eglwkdot-wandb-metadata.json
+2024-12-17 22:17:08,323 INFO    wandb-upload_2:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/7k7jg9di-diff.patch
+2024-12-17 22:17:08,349 INFO    HandlerThread:84579 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/tb"
+  save: true
+}
+control {
+  relay_id: "81d3cf2cbfed43cbb7d230993cc9c89e"
+}
+_info {
+  stream_id: "sqbcnf7o"
+}
+
+2024-12-17 22:17:08,349 DEBUG   HandlerThread:84579 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-17 22:17:08,395 INFO    wandb-upload_1:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/ggysc832-code/FISH/eval_robot.py
+2024-12-17 22:17:08,456 DEBUG   SenderThread:84579 [sender.py:send():379] send: telemetry
+2024-12-17 22:17:08,456 DEBUG   SenderThread:84579 [sender.py:send():379] send: tbrecord
+2024-12-17 22:17:08,456 DEBUG   SenderThread:84579 [sender.py:send():379] send: files
+2024-12-17 22:17:08,457 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0 with policy live
+2024-12-17 22:17:08,631 INFO    wandb-upload_0:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/hghsffpo-events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0
+2024-12-17 22:17:08,902 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0
+2024-12-17 22:17:08,903 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/diff.patch
+2024-12-17 22:17:08,903 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/code/FISH/eval_robot.py
+2024-12-17 22:17:08,903 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-metadata.json
+2024-12-17 22:17:08,903 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:08,903 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/requirements.txt
+2024-12-17 22:17:08,903 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/code/FISH
+2024-12-17 22:17:08,903 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/code
+2024-12-17 22:17:09,302 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:10,302 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:10,902 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:11,302 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:12,302 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:13,302 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:17:13,302 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:13,845 DEBUG   SenderThread:84579 [sender.py:send():379] send: config
+2024-12-17 22:17:14,303 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:14,903 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:15,303 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:16,303 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:17,303 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:18,302 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:17:18,303 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:18,904 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:19,303 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:20,304 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:20,904 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:21,304 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:22,304 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:23,301 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:17:23,301 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:17:23,304 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:23,434 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:17:24,304 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:24,906 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:25,304 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:26,304 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:26,906 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:27,304 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:28,305 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:28,457 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:17:29,305 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:30,305 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:30,907 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:31,305 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:32,306 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:32,908 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:33,306 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:34,303 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:17:34,306 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:35,306 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:36,306 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:36,909 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:37,306 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:38,301 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:17:38,302 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:17:38,345 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:39,307 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:40,117 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:17:40,306 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:40,910 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:40,910 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/config.yaml
+2024-12-17 22:17:41,307 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:42,307 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:43,307 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:44,307 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:44,911 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:45,303 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:17:45,307 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:46,307 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:47,307 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:48,307 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:49,308 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:50,308 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:51,113 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:17:51,308 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:52,310 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:52,913 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:53,301 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:17:53,302 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:17:53,345 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:54,310 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:55,329 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:56,211 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:17:56,225 DEBUG   SenderThread:84579 [sender.py:send():379] send: files
+2024-12-17 22:17:56,225 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_886132ad98d0c31f9311.mp4 with policy now
+2024-12-17 22:17:56,225 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 22:17:56,226 DEBUG   SenderThread:84579 [sender.py:send():379] send: history
+2024-12-17 22:17:56,226 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:17:56,228 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:17:56,311 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:56,485 INFO    wandb-upload_1:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/uq7df1o8-media/videos/eval/0_eval_0_886132ad98d0c31f9311.mp4
+2024-12-17 22:17:56,915 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:56,915 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json
+2024-12-17 22:17:56,915 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/0_eval_0_886132ad98d0c31f9311.mp4
+2024-12-17 22:17:56,915 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media
+2024-12-17 22:17:56,915 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval
+2024-12-17 22:17:56,915 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos
+2024-12-17 22:17:57,186 INFO    wandb-upload_0:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/ucxh4z2j-media/videos/eval/0_eval_0_886132ad98d0c31f9311.mp4
+2024-12-17 22:17:57,311 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:58,312 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:17:58,915 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:17:59,311 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:00,311 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:00,916 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:18:01,311 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:01,477 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:18:02,311 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:03,311 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:04,311 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:04,917 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0
+2024-12-17 22:18:04,918 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:18:05,312 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:06,312 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:06,918 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:18:07,305 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:18:07,312 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:07,975 DEBUG   SystemMonitor:84579 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-17 22:18:07,976 DEBUG   SenderThread:84579 [sender.py:send():379] send: stats
+2024-12-17 22:18:08,302 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:18:08,302 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:18:08,312 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:08,919 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:18:09,312 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:10,312 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:10,920 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:18:11,312 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:12,312 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:12,370 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:18:13,312 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:14,313 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:15,313 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:16,313 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:17,313 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:18,305 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:18:18,313 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:19,313 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:20,313 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:21,313 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:22,313 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:23,302 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:18:23,302 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:18:23,345 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:23,435 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:18:24,314 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:25,314 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:26,314 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:27,314 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:28,314 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:29,305 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:18:29,314 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:30,315 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:31,315 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:32,315 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:33,315 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:34,315 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:34,717 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:18:35,315 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:35,940 DEBUG   SenderThread:84579 [sender.py:send():379] send: files
+2024-12-17 22:18:35,940 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_c21bfe5dd150e694f7bd.mp4 with policy now
+2024-12-17 22:18:35,942 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 22:18:35,943 DEBUG   SenderThread:84579 [sender.py:send():379] send: history
+2024-12-17 22:18:35,943 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:18:35,944 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:18:36,188 INFO    wandb-upload_1:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/4cgqz4qo-media/videos/eval/1_eval_1_c21bfe5dd150e694f7bd.mp4
+2024-12-17 22:18:36,319 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:36,928 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:18:36,928 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json
+2024-12-17 22:18:36,928 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/1_eval_1_c21bfe5dd150e694f7bd.mp4
+2024-12-17 22:18:36,928 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval
+2024-12-17 22:18:37,214 INFO    wandb-upload_0:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/9sjtqb51-media/videos/eval/1_eval_1_c21bfe5dd150e694f7bd.mp4
+2024-12-17 22:18:37,318 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:37,976 DEBUG   SenderThread:84579 [sender.py:send():379] send: stats
+2024-12-17 22:18:38,306 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:18:38,307 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:18:38,349 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:39,332 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:39,791 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:18:40,335 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:40,929 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:18:41,338 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:42,335 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:43,336 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:44,339 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:44,930 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:18:45,309 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:18:45,337 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:46,336 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:47,335 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:48,348 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:49,350 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:50,328 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:18:50,340 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:51,343 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:52,352 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:52,739 DEBUG   SenderThread:84579 [sender.py:send():379] send: history
+2024-12-17 22:18:52,739 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:18:52,740 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:18:52,933 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0
+2024-12-17 22:18:52,933 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json
+2024-12-17 22:18:53,180 INFO    wandb-upload_1:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/qz8moi18-events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0
+2024-12-17 22:18:53,305 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:18:53,305 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:18:53,340 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:54,353 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:54,934 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:18:55,356 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:55,397 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:18:56,352 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:56,934 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:18:57,348 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:58,348 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:18:58,935 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:18:59,348 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:00,349 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:01,311 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:19:01,349 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:02,348 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:03,348 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:04,350 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:05,349 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:06,312 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:19:06,350 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:07,351 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:07,976 DEBUG   SenderThread:84579 [sender.py:send():379] send: stats
+2024-12-17 22:19:08,306 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:19:08,307 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:19:08,350 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:09,350 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:10,351 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:11,351 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:11,381 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:19:12,352 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:13,352 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:14,352 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:15,352 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:16,353 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:17,313 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:19:17,353 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:18,353 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:19,353 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:20,354 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:21,354 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:22,357 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:23,310 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:19:23,310 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:19:23,357 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:23,385 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:19:24,355 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:25,368 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:25,684 DEBUG   SenderThread:84579 [sender.py:send():379] send: files
+2024-12-17 22:19:25,684 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_312f8f530abaf399946b.mp4 with policy now
+2024-12-17 22:19:25,689 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 22:19:25,690 DEBUG   SenderThread:84579 [sender.py:send():379] send: history
+2024-12-17 22:19:25,690 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:19:25,691 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:19:25,933 INFO    wandb-upload_0:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/qyjuq5lx-media/videos/eval/2_eval_2_312f8f530abaf399946b.mp4
+2024-12-17 22:19:25,944 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json
+2024-12-17 22:19:25,944 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/2_eval_2_312f8f530abaf399946b.mp4
+2024-12-17 22:19:25,944 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval
+2024-12-17 22:19:26,186 INFO    wandb-upload_1:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/e6gxg7js-media/videos/eval/2_eval_2_312f8f530abaf399946b.mp4
+2024-12-17 22:19:26,367 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:26,944 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:19:27,375 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:28,377 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:29,316 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:19:29,382 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:30,379 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:30,946 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:19:31,379 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:32,413 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:33,390 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:34,389 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:34,947 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:19:35,316 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:19:35,391 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:36,398 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:36,589 DEBUG   SenderThread:84579 [sender.py:send():379] send: history
+2024-12-17 22:19:36,589 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:19:36,590 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:19:36,948 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0
+2024-12-17 22:19:36,948 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json
+2024-12-17 22:19:37,394 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:37,977 DEBUG   SenderThread:84579 [sender.py:send():379] send: stats
+2024-12-17 22:19:38,351 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:19:38,352 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:19:38,401 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:38,948 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:19:39,407 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:40,403 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:40,949 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:19:41,045 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:19:41,408 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:42,399 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:43,416 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:44,399 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:44,950 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:19:45,399 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:46,399 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:46,505 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:19:47,399 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:48,399 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:49,399 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:50,399 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:51,400 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:52,316 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:19:52,400 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:53,323 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:19:53,323 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:19:53,400 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:54,400 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:55,400 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:56,400 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:57,400 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:57,427 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:19:58,400 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:19:59,400 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:00,400 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:01,400 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:02,401 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:03,316 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:20:03,401 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:04,404 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:05,402 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:06,402 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:07,402 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:07,977 DEBUG   SenderThread:84579 [sender.py:send():379] send: stats
+2024-12-17 22:20:08,316 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:20:08,323 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:20:08,323 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:20:08,402 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:09,403 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:10,403 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:10,959 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:20:11,130 DEBUG   SenderThread:84579 [sender.py:send():379] send: files
+2024-12-17 22:20:11,130 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_91b538bdbc0a7862dd54.mp4 with policy now
+2024-12-17 22:20:11,135 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 22:20:11,135 DEBUG   SenderThread:84579 [sender.py:send():379] send: history
+2024-12-17 22:20:11,135 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:20:11,136 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:20:11,398 INFO    wandb-upload_0:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/6c53q71e-media/videos/eval/3_eval_3_91b538bdbc0a7862dd54.mp4
+2024-12-17 22:20:11,416 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:11,959 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json
+2024-12-17 22:20:11,959 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/3_eval_3_91b538bdbc0a7862dd54.mp4
+2024-12-17 22:20:11,960 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval
+2024-12-17 22:20:12,225 INFO    wandb-upload_1:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/g46ird7n-media/videos/eval/3_eval_3_91b538bdbc0a7862dd54.mp4
+2024-12-17 22:20:12,408 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:12,960 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:20:13,327 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:20:13,415 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:14,427 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:15,426 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:16,424 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:16,961 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:20:17,424 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:18,428 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:18,962 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:20:19,322 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:20:19,464 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:20,433 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:21,435 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:22,435 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:23,328 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:20:23,329 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:20:23,444 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:24,406 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:20:24,450 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:25,449 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:26,461 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:27,464 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:27,673 DEBUG   SenderThread:84579 [sender.py:send():379] send: history
+2024-12-17 22:20:27,673 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:20:27,673 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:20:27,965 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0
+2024-12-17 22:20:27,965 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json
+2024-12-17 22:20:28,200 INFO    wandb-upload_0:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/hjkbjwos-events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0
+2024-12-17 22:20:28,457 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:28,965 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:20:29,457 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:30,336 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:20:30,457 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:30,966 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:20:31,457 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:32,459 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:32,966 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:20:33,467 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:34,457 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:34,967 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:20:35,457 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:36,162 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:20:36,457 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:36,968 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:20:37,458 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:37,978 DEBUG   SenderThread:84579 [sender.py:send():379] send: stats
+2024-12-17 22:20:38,328 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:20:38,328 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:20:38,458 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:39,459 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:40,459 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:41,459 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:41,472 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:20:42,459 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:43,459 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:44,459 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:45,460 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:46,459 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:47,333 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:20:47,459 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:48,460 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:49,461 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:50,461 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:51,461 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:52,333 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:20:52,461 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:53,331 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:20:53,331 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:20:53,463 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:54,463 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:55,463 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:56,462 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:57,470 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:57,505 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:20:58,463 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:20:58,975 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:20:59,463 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:00,464 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:01,463 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:02,493 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:02,529 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:21:02,795 DEBUG   SenderThread:84579 [sender.py:send():379] send: files
+2024-12-17 22:21:02,795 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_fdb67280e4956e7c7ab4.mp4 with policy now
+2024-12-17 22:21:02,798 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 22:21:02,798 DEBUG   SenderThread:84579 [sender.py:send():379] send: history
+2024-12-17 22:21:02,799 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:21:02,799 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:21:02,976 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:21:02,976 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json
+2024-12-17 22:21:02,977 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/4_eval_4_fdb67280e4956e7c7ab4.mp4
+2024-12-17 22:21:02,977 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval
+2024-12-17 22:21:03,064 INFO    wandb-upload_1:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/jpil4fgf-media/videos/eval/4_eval_4_fdb67280e4956e7c7ab4.mp4
+2024-12-17 22:21:03,333 INFO    wandb-upload_0:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/qluoymh3-media/videos/eval/4_eval_4_fdb67280e4956e7c7ab4.mp4
+2024-12-17 22:21:03,488 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:04,481 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:04,977 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:21:05,487 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:06,490 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:07,482 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:07,701 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:21:07,978 DEBUG   SenderThread:84579 [sender.py:send():379] send: stats
+2024-12-17 22:21:08,354 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:21:08,354 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:21:08,483 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:08,979 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:21:09,481 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:10,489 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:10,980 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:21:11,453 DEBUG   SenderThread:84579 [sender.py:send():379] send: exit
+2024-12-17 22:21:11,454 INFO    SenderThread:84579 [sender.py:send_exit():586] handling exit code: 0
+2024-12-17 22:21:11,454 INFO    SenderThread:84579 [sender.py:send_exit():588] handling runtime: 243
+2024-12-17 22:21:11,454 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:21:11,455 INFO    SenderThread:84579 [sender.py:send_exit():594] send defer
+2024-12-17 22:21:11,455 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:11,455 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-17 22:21:11,455 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:11,455 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-17 22:21:11,455 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 1
+2024-12-17 22:21:11,455 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:11,455 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-17 22:21:11,456 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:11,456 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-17 22:21:11,456 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 2
+2024-12-17 22:21:11,456 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:11,456 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-17 22:21:11,456 INFO    HandlerThread:84579 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-17 22:21:11,456 DEBUG   SystemMonitor:84579 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-17 22:21:11,456 INFO    HandlerThread:84579 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-17 22:21:11,456 DEBUG   SystemMonitor:84579 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-17 22:21:11,456 INFO    HandlerThread:84579 [interfaces.py:finish():200] Joined disk monitor
+2024-12-17 22:21:11,502 INFO    HandlerThread:84579 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-17 22:21:11,502 INFO    HandlerThread:84579 [interfaces.py:finish():200] Joined memory monitor
+2024-12-17 22:21:11,502 INFO    HandlerThread:84579 [interfaces.py:finish():200] Joined network monitor
+2024-12-17 22:21:11,503 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:11,503 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-17 22:21:11,503 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 3
+2024-12-17 22:21:11,503 DEBUG   SenderThread:84579 [sender.py:send():379] send: stats
+2024-12-17 22:21:11,503 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:11,503 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-17 22:21:11,503 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:11,503 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-17 22:21:11,503 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 4
+2024-12-17 22:21:11,503 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:11,503 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-17 22:21:11,980 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0
+2024-12-17 22:21:11,980 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json
+2024-12-17 22:21:12,981 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:21:17,505 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:21:17,505 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:21:17,505 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:17,506 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-17 22:21:17,506 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-17 22:21:17,506 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 5
+2024-12-17 22:21:17,506 DEBUG   SenderThread:84579 [sender.py:send():379] send: history
+2024-12-17 22:21:17,506 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:17,506 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:21:17,506 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-17 22:21:17,507 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:21:17,507 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:21:17,507 DEBUG   SenderThread:84579 [sender.py:send():379] send: history
+2024-12-17 22:21:17,507 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:21:17,511 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:21:17,511 DEBUG   SenderThread:84579 [sender.py:send():379] send: summary
+2024-12-17 22:21:17,511 INFO    SenderThread:84579 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:21:17,511 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:17,512 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-17 22:21:17,512 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 6
+2024-12-17 22:21:17,512 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:17,512 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-17 22:21:17,512 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:17,512 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-17 22:21:17,512 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 7
+2024-12-17 22:21:17,512 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:21:17,512 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:17,512 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-17 22:21:17,512 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:17,513 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-17 22:21:17,982 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json
+2024-12-17 22:21:18,454 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:21:18,708 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 8
+2024-12-17 22:21:18,708 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:21:18,708 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:18,709 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-17 22:21:18,709 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:18,709 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-17 22:21:18,709 INFO    SenderThread:84579 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-17 22:21:18,715 INFO    SenderThread:84579 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-17 22:21:18,731 INFO    SenderThread:84579 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-17 22:21:18,743 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 9
+2024-12-17 22:21:18,743 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:18,743 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-17 22:21:18,743 DEBUG   SenderThread:84579 [sender.py:send():379] send: artifact
+2024-12-17 22:21:18,983 INFO    Thread-12 :84579 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:21:19,455 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:21:19,527 INFO    wandb-upload_2:84579 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpqschr5qi
+2024-12-17 22:21:19,527 INFO    wandb-upload_0:84579 [upload_job.py:push():85] Skipped uploading /tmp/tmpsbqi9vuh/wandb-job.json
+2024-12-17 22:21:19,649 INFO    wandb-upload_1:84579 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpfksjk6nr
+2024-12-17 22:21:20,210 INFO    SenderThread:84579 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM4MzQxMDY4OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM4MzM4ODU2OA==', 'versionIndex': 383}}}
+2024-12-17 22:21:20,210 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:20,210 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-17 22:21:20,210 INFO    SenderThread:84579 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-17 22:21:20,984 INFO    SenderThread:84579 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files
+2024-12-17 22:21:20,984 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/config.yaml config.yaml
+2024-12-17 22:21:20,984 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json wandb-summary.json
+2024-12-17 22:21:20,984 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/requirements.txt requirements.txt
+2024-12-17 22:21:20,984 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0 events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0
+2024-12-17 22:21:20,987 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-metadata.json wandb-metadata.json
+2024-12-17 22:21:20,987 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log output.log
+2024-12-17 22:21:20,987 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/diff.patch diff.patch
+2024-12-17 22:21:20,989 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-17 22:21:20,989 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/2_eval_2_312f8f530abaf399946b.mp4 media/videos/eval/2_eval_2_312f8f530abaf399946b.mp4
+2024-12-17 22:21:20,989 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/0_eval_0_886132ad98d0c31f9311.mp4 media/videos/eval/0_eval_0_886132ad98d0c31f9311.mp4
+2024-12-17 22:21:20,989 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/3_eval_3_91b538bdbc0a7862dd54.mp4 media/videos/eval/3_eval_3_91b538bdbc0a7862dd54.mp4
+2024-12-17 22:21:20,989 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/4_eval_4_fdb67280e4956e7c7ab4.mp4 media/videos/eval/4_eval_4_fdb67280e4956e7c7ab4.mp4
+2024-12-17 22:21:20,989 INFO    SenderThread:84579 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/media/videos/eval/1_eval_1_c21bfe5dd150e694f7bd.mp4 media/videos/eval/1_eval_1_c21bfe5dd150e694f7bd.mp4
+2024-12-17 22:21:20,989 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 10
+2024-12-17 22:21:20,989 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:21:20,989 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:20,990 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-17 22:21:20,992 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:20,992 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-17 22:21:20,992 INFO    SenderThread:84579 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 22:21:21,167 INFO    wandb-upload_0:84579 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/wandb-summary.json
+2024-12-17 22:21:21,200 INFO    wandb-upload_1:84579 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/requirements.txt
+2024-12-17 22:21:21,217 INFO    wandb-upload_2:84579 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/config.yaml
+2024-12-17 22:21:21,265 INFO    wandb-upload_3:84579 [upload_job.py:push():130] Uploaded file /tmp/tmpf6cm3vldwandb/r373gfgp-events.out.tfevents.1734491828.leonmkim-ROG-Strix-G15CS-G15CS.84391.0
+2024-12-17 22:21:21,320 INFO    wandb-upload_4:84579 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/files/output.log
+2024-12-17 22:21:21,456 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:21:21,456 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:21:21,521 INFO    Thread-11 (_thread_body):84579 [sender.py:transition_state():614] send defer: 11
+2024-12-17 22:21:21,521 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:21,521 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-17 22:21:21,521 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:21,521 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-17 22:21:21,521 INFO    SenderThread:84579 [file_pusher.py:join():175] waiting for file pusher
+2024-12-17 22:21:21,522 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 12
+2024-12-17 22:21:21,522 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:21,522 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-17 22:21:21,522 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:21,523 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-17 22:21:21,523 INFO    SenderThread:84579 [file_stream.py:finish():601] file stream finish called
+2024-12-17 22:21:21,596 INFO    SenderThread:84579 [file_stream.py:finish():605] file stream finish is done
+2024-12-17 22:21:21,596 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 13
+2024-12-17 22:21:21,597 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:21,597 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-17 22:21:21,597 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:21,597 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-17 22:21:21,597 INFO    SenderThread:84579 [sender.py:transition_state():614] send defer: 14
+2024-12-17 22:21:21,597 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:21:21,597 DEBUG   SenderThread:84579 [sender.py:send():379] send: final
+2024-12-17 22:21:21,597 INFO    HandlerThread:84579 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-17 22:21:21,597 DEBUG   SenderThread:84579 [sender.py:send():379] send: footer
+2024-12-17 22:21:21,597 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:21:21,598 INFO    SenderThread:84579 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-17 22:21:21,598 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:21:21,598 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:21:21,598 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: server_info
+2024-12-17 22:21:21,598 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: server_info
+2024-12-17 22:21:21,600 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:21:21,600 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-17 22:21:21,600 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-17 22:21:21,600 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:21:21,661 DEBUG   SenderThread:84579 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:21:21,662 INFO    MainThread:84579 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-17 22:21:21,662 INFO    MainThread:84579 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-17 22:21:21,662 INFO    MainThread:84579 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-17 22:21:21,662 DEBUG   HandlerThread:84579 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-17 22:21:21,662 INFO    HandlerThread:84579 [handler.py:finish():882] shutting down handler
+2024-12-17 22:21:22,600 INFO    WriterThread:84579 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/run-sqbcnf7o.wandb
+2024-12-17 22:21:22,662 INFO    SenderThread:84579 [sender.py:finish():1615] shutting down sender
+2024-12-17 22:21:22,662 INFO    SenderThread:84579 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 22:21:22,662 INFO    SenderThread:84579 [file_pusher.py:join():175] waiting for file pusher

221701/wandb/run-20241217_221707-sqbcnf7o/logs/debug.log

@@ -0,0 +1,31 @@
+2024-12-17 22:17:07,582 INFO    MainThread:84391 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-17 22:17:07,582 INFO    MainThread:84391 [wandb_setup.py:_flush():76] Configure stats pid to 84391
+2024-12-17 22:17:07,582 INFO    MainThread:84391 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-17 22:17:07,582 INFO    MainThread:84391 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/settings
+2024-12-17 22:17:07,582 INFO    MainThread:84391 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-17 22:17:07,583 INFO    MainThread:84391 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-17 22:17:07,583 INFO    MainThread:84391 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-17 22:17:07,583 INFO    MainThread:84391 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-17 22:17:07,583 INFO    MainThread:84391 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/logs/debug.log
+2024-12-17 22:17:07,583 INFO    MainThread:84391 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/wandb/run-20241217_221707-sqbcnf7o/logs/debug-internal.log
+2024-12-17 22:17:07,583 INFO    MainThread:84391 [wandb_init.py:init():569] calling init triggers
+2024-12-17 22:17:07,583 INFO    MainThread:84391 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '511_0', 'wandb_notes': '511_0_', 'eval': True, 'true_action_history': False, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': False, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['electrodyn_twodim'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/511_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701'}
+2024-12-17 22:17:07,584 INFO    MainThread:84391 [wandb_init.py:init():619] starting backend
+2024-12-17 22:17:07,584 INFO    MainThread:84391 [wandb_init.py:init():623] setting up manager
+2024-12-17 22:17:07,588 INFO    MainThread:84391 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-17 22:17:07,589 INFO    MainThread:84391 [wandb_init.py:init():631] backend started and connected
+2024-12-17 22:17:07,600 INFO    MainThread:84391 [wandb_init.py:init():720] updated telemetry
+2024-12-17 22:17:07,606 INFO    MainThread:84391 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-17 22:17:07,920 INFO    MainThread:84391 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-17 22:17:07,962 INFO    MainThread:84391 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-17 22:17:07,962 INFO    MainThread:84391 [wandb_init.py:init():804] starting run threads in backend
+2024-12-17 22:17:08,301 INFO    MainThread:84391 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-17 22:17:08,301 INFO    MainThread:84391 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-17 22:17:08,301 INFO    MainThread:84391 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-17 22:17:08,303 INFO    MainThread:84391 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-17 22:17:08,305 INFO    MainThread:84391 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-17 22:17:08,305 INFO    MainThread:84391 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/511_0/221701/tb, True
+2024-12-17 22:17:13,844 INFO    MainThread:84391 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'electrodyn', 'left_book_slot': 'twodim'}
+2024-12-17 22:21:22,664 WARNING MsgRouterThr:84391 [router.py:message_loop():77] message_loop has been closed

config.yaml

@@ -0,0 +1,546 @@
+root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data
+replay_buffer_size: 150000
+replay_buffer_num_workers: 2
+nstep: 3
+batch_size: 128
+seed: 0
+dataset_shuffle_seed: 0
+device: cuda
+save_video: true
+save_train_video: true
+use_tb: true
+use_wandb: true
+wandb_run_id: '511_0'
+wandb_notes: '511_0_'
+eval: false
+true_action_history: false
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.2
+env_dtc_max_value: 0.4
+grasped_normals_mask_max_dtc_value: 0.2
+env_normals_mask_max_dtc_value: 0.4
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: true
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type: small
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+baseline: false
+train_demo_idxs_list_or_num: -1
+log_train_every_steps: 25
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+store_dataset_in_memory: false
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+include_groups_list:
+- electrodyn_twodim
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+name_of_valid_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr
+valid_demo_idxs_list_or_num: null
+val_num_groups: 0
+load_bc: ${agent.load_checkpoint}
+checkpoint_epoch_list:
+- 99
+- 199
+- 299
+- 399
+- 499
+- 599
+- 699
+- 799
+- 899
+- 999
+- 1249
+- 1499
+- 1749
+- 1999
+- 2999
+- 3999
+- 4999
+- 5999
+- 6999
+- 7999
+- 8999
+- 9999
+snapshot_root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data/FISH
+save_snapshot: true
+save_last_snapshot: true
+save_snapshot_when_done: true
+top_k_checkpoints: 5
+save_snapshot_link_to_weights_dir: deprecated
+bc_regularize: false
+bc_weight_type: qfilter
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: 6
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+      input_shapes:
+        observation.image:
+        - 13
+        - 180
+        - 240
+        context_observation.image:
+        - 13
+        - 180
+        - 240
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: cosine
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: 0.0001
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 1000000
+  num_seed_frames: 0
+  num_train_epochs: 15000
+  validate_every_epochs: 250
+  validate_diffusion_on_action_loss_every_epochs: 250
+  train_eval_diffusion_on_action_loss_every_epochs: 250
+  check_topk_every_epochs: 250
+  save_snapshot_every_epochs: 1500
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: ${device}
+      cam_resize_shape: ${agent.desired_image_shape}
+      orig_cam_shape: ${agent.orig_cam_shape}
+      policy_frequency: ${policy_frequency}
+      interpolation_frequency: ${interpolation_frequency}
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: ${agent.config.policy_cfg.horizon}
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: ${agent.config.policy_cfg.kernel_size}
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: ${num_train_frames_diffusion}
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: ${use_depth}
+        use_color: ${use_color}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_masks}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${color_crop_type}
+          depth_crop_type: ${depth_crop_type}
+          segmask_crop_type: ${segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${add_crop_binary_mask}
+          add_coord_conv_map: ${add_coord_conv_map}
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: ${use_context_color}
+          use_depth: ${use_context_depth}
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: ${use_context_segmask}
+            representation: ${mask_representation}
+            mask_list: ${mask_list}
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: ${context_color_crop_type}
+            depth_crop_type: ${context_depth_crop_type}
+            segmask_crop_type: ${context_segmask_crop_type}
+            crop_hw: ${crop_hw}
+            crop_down_offset: ${crop_down_offset}
+            add_crop_binary_mask: ${context_add_crop_binary_mask}
+            add_coord_conv_map: ${context_add_coord_conv_map}
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: ${use_contact_map}
+        use_sdf_maps: ${use_sdf_maps}
+        use_normals_maps: ${use_normals_maps}
+        which_objects: ${which_objects}
+        grasped_dtc_max_value: ${grasped_dtc_max_value}
+        env_dtc_max_value: ${env_dtc_max_value}
+        grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+        env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+        clamp_dtc: ${clamp_dtc}
+        max_contact_prob: ${max_contact_prob}
+        mask_normals_within_sdf: ${mask_normals_within_sdf}
+        dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+        adaptive_normals_mask: ${adaptive_normals_mask}
+        max_depth: ${max_depth}
+        image_shape: ${agent.desired_image_shape}
+        learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: ${contact_model_name}
+        zero_centered: false
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 15000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 250
+validate_diffusion_on_action_loss_every_epochs: 250
+train_eval_diffusion_on_action_loss_every_epochs: 250
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: 250
+save_snapshot_every_epochs_diffusion: 1500
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1