Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,18 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/1_eval_1_8a67326b5dfefa361130.mp4 filter=lfs diff=lfs merge=lfs -text
+220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/2_eval_2_4735c6188bf9f36869cb.mp4 filter=lfs diff=lfs merge=lfs -text
+220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/3_eval_3_a4af24fd7d85bfc8c826.mp4 filter=lfs diff=lfs merge=lfs -text
+220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/0_eval_0_f8fef5727eff9ff71143.mp4 filter=lfs diff=lfs merge=lfs -text
+220948/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/4_eval_4_ccefe4eb2a70fd4856d6.mp4 filter=lfs diff=lfs merge=lfs -text
+220948/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+220948/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+220948/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+220948/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text
+220948/episode_rosbags/episode_2_2024-12-17-22-11-39.bag filter=lfs diff=lfs merge=lfs -text
+220948/episode_rosbags/episode_0_2024-12-17-22-10-14.bag filter=lfs diff=lfs merge=lfs -text
+220948/episode_rosbags/episode_1_2024-12-17-22-10-55.bag filter=lfs diff=lfs merge=lfs -text
+220948/episode_rosbags/episode_4_2024-12-17-22-13-12.bag filter=lfs diff=lfs merge=lfs -text
+220948/episode_rosbags/episode_3_2024-12-17-22-12-20.bag filter=lfs diff=lfs merge=lfs -text

220945/.hydra/config.yaml

@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '510_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${now:%H%M%S}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

220945/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_policy
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220945
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

220945/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion

220945/eval_policy.log

@@ -0,0 +1,15 @@
+[2024-12-17 22:09:45,888][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-17 22:09:45,891][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+
+[2024-12-17 22:09:49,730][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  payload = torch.load(f)
+

220948/.hydra/config.yaml

@@ -0,0 +1,350 @@
+root_dir: /home/${oc.env:USER}/fish_leon
+nstep: 3
+seed: 41
+dataset_shuffle_seed: ${seed}
+device: cuda
+save_video: true
+save_buffer: true
+use_tb: true
+baseline: false
+use_wandb: true
+eval: true
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.105
+env_dtc_max_value: 0.425
+grasped_normals_mask_max_dtc_value: 0.105
+env_normals_mask_max_dtc_value: 0.425
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: false
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+num_eval: 5
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+random_start: false
+eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name}
+train_demo_idxs_list_or_num: null
+num_valid_demos: null
+val_num_groups: 3
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+bc_regularize: false
+bc_weight_type: qfilter
+load_checkpoint: ${agent.load_checkpoint}
+wandb_run_id: '510_0'
+true_action_history: false
+wandb_notes: null
+checkpoint_epoch: 12000
+load_residual_weight: false
+checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH
+checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+final_experiment_dir: ${experiment_dir}/${now:%H%M%S}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      input_shapes:
+        observation.image: ${agent.config.cam_resize_shape}
+        context_observation.image: ${agent.config.cam_resize_shape}
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: ${agent.config.policy_cfg.n_action_steps}
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: constant
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: ${agent.config.train_cfg.lr}
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 2010
+  num_seed_frames: 260
+  num_train_epochs: 5000
+  validate_every_epochs: 100
+  validate_diffusion_on_action_loss_every_epochs: 500
+  train_eval_diffusion_on_action_loss_every_epochs: 500
+  check_topk_every_epochs: 10
+  save_snapshot_every_epochs: 5000
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg: ???
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 5000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 25
+validate_diffusion_on_action_loss_every_epochs: 50
+train_eval_diffusion_on_action_loss_every_epochs: 500
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs}
+save_snapshot_every_epochs_diffusion: 5000
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy

220948/.hydra/hydra.yaml

@@ -0,0 +1,169 @@
+hydra:
+  run:
+    dir: ${final_experiment_dir}
+  sweep:
+    dir: ${final_experiment_dir}
+    subdir: ${hydra.job.num}
+  launcher:
+    submitit_folder: ${final_experiment_dir}/.slurm
+    timeout_min: 60
+    cpus_per_task: null
+    gpus_per_node: null
+    tasks_per_node: 1
+    mem_gb: null
+    nodes: 1
+    name: ${hydra.job.name}
+    stderr_to_stdout: false
+    _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - agent=diffusion
+    - suite=frankagym
+    - suite/frankagym_task@_global_=insertion
+  job:
+    name: eval_robot
+    chdir: true
+    override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym
+    id: ???
+    num: ???
+    config_name: config_eval
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /home/leonmkim/fish_leon/FISH
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/leonmkim/fish_leon/FISH/cfgs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948
+    choices:
+      suite: frankagym
+      suite/frankagym_task@_global_: insertion
+      agent: diffusion
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: submitit_local
+      hydra/output: default
+  verbose: false

220948/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- agent=diffusion
+- suite=frankagym
+- suite/frankagym_task@_global_=insertion

220948/episode_rosbags/aligned_depth_to_color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

220948/episode_rosbags/cam_tf_world.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e
+size 256

220948/episode_rosbags/color_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55
+size 200

220948/episode_rosbags/depth_K.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7
+size 200

220948/episode_rosbags/episode_0_2024-12-17-22-10-14.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2b087cd5359a837be6034714a39aeaa789f98fb4c31aaa19b5772cb8c979c80
+size 1330382964

220948/episode_rosbags/episode_1_2024-12-17-22-10-55.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a038bdb04d7bdcc31122c5d5ef3ce4ce795488e2c9a5b122a11190c12a597d69
+size 1333650048

220948/episode_rosbags/episode_2_2024-12-17-22-11-39.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73dd1be745550f3dc895aa668d19b63211dbea4e5ebd15ffba1a94f379fca045
+size 1333847287

220948/episode_rosbags/episode_3_2024-12-17-22-12-20.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17d8c1b8fb98339b024865da6c5f17ebf1ee937ca1089fc36b60f3ed556a0969
+size 1335329172

220948/episode_rosbags/episode_4_2024-12-17-22-13-12.bag

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c5fd82c7e9b68da3405e0f74f4d424b7d0aa6b6e572e0b64362a54dea740811
+size 1333689386

220948/eval_robot.log

@@ -0,0 +1,12 @@
+[2024-12-17 22:09:48,382][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  @hydra.main(config_path='cfgs', config_name='config_eval')
+
+[2024-12-17 22:09:48,385][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: 
+The version_base parameter is not specified.
+Please specify a compatability version level, or None.
+Will assume defaults for version 1.1
+  hydra.initialize(
+

220948/eval_video/0_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8fef5727eff9ff711434a6872b2606d4abe372028bd219ceb58dfa95c013ee2
+size 1152439

220948/eval_video/1_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a67326b5dfefa3611301ecb0f2a2100e831e8d33dfa72fcc6d707320e8bfc17
+size 1160665

220948/eval_video/2_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4735c6188bf9f36869cb28f87bdc9cb4676365377b2b1bd34c01893841b876db
+size 1227110

220948/eval_video/3_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4af24fd7d85bfc8c826ed9fb0677ef3b356f3a74873bf3e4f95489ce5d37e74
+size 1237139

220948/eval_video/4_eval.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccefe4eb2a70fd4856d6e54f10fa162d2eb75a982b20f5d9e7ca1f796e073ab1
+size 1181573

220948/tb/events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:831e796de7de833e3a9c3f9a056a250abbadb5c182d6f5d03eb2629f93876e12
+size 1103

220948/wandb/debug-internal.log

@@ -0,0 +1,680 @@
+2024-12-17 22:09:54,394 INFO    StreamThr :82813 [internal.py:wandb_internal():85] W&B internal server running at pid: 82813, started at: 2024-12-17 22:09:54.393760
+2024-12-17 22:09:54,395 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status
+2024-12-17 22:09:54,410 INFO    WriterThread:82813 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/run-7lp0oxke.wandb
+2024-12-17 22:09:54,410 DEBUG   SenderThread:82813 [sender.py:send():379] send: header
+2024-12-17 22:09:54,418 DEBUG   SenderThread:82813 [sender.py:send():379] send: run
+2024-12-17 22:09:54,634 INFO    SenderThread:82813 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files
+2024-12-17 22:09:54,634 INFO    SenderThread:82813 [sender.py:_start_run_threads():1188] run started: 7lp0oxke with start time 1734491394.394366
+2024-12-17 22:09:54,650 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: check_version
+2024-12-17 22:09:54,650 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: check_version
+2024-12-17 22:09:54,721 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: run_start
+2024-12-17 22:09:54,725 DEBUG   HandlerThread:82813 [system_info.py:__init__():26] System info init
+2024-12-17 22:09:54,726 DEBUG   HandlerThread:82813 [system_info.py:__init__():41] System info init done
+2024-12-17 22:09:54,726 INFO    HandlerThread:82813 [system_monitor.py:start():194] Starting system monitor
+2024-12-17 22:09:54,726 INFO    SystemMonitor:82813 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-17 22:09:54,726 INFO    HandlerThread:82813 [system_monitor.py:probe():214] Collecting system info
+2024-12-17 22:09:54,727 INFO    SystemMonitor:82813 [interfaces.py:start():188] Started cpu monitoring
+2024-12-17 22:09:54,727 INFO    SystemMonitor:82813 [interfaces.py:start():188] Started disk monitoring
+2024-12-17 22:09:54,727 INFO    SystemMonitor:82813 [interfaces.py:start():188] Started gpu monitoring
+2024-12-17 22:09:54,728 INFO    SystemMonitor:82813 [interfaces.py:start():188] Started memory monitoring
+2024-12-17 22:09:54,728 INFO    SystemMonitor:82813 [interfaces.py:start():188] Started network monitoring
+2024-12-17 22:09:54,739 DEBUG   HandlerThread:82813 [system_info.py:probe():152] Probing system
+2024-12-17 22:09:54,741 DEBUG   HandlerThread:82813 [system_info.py:_probe_git():137] Probing git
+2024-12-17 22:09:54,745 DEBUG   HandlerThread:82813 [system_info.py:_probe_git():145] Probing git done
+2024-12-17 22:09:54,745 DEBUG   HandlerThread:82813 [system_info.py:probe():200] Probing system done
+2024-12-17 22:09:54,745 DEBUG   HandlerThread:82813 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-18T03:09:54.739757', 'startedAt': '2024-12-18T03:09:54.386675', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '5f28b54373d7d41e4ebace380ff1080a8ffc4051'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3563.9849999999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4511.879, 'min': 800.0, 'max': 4700.0}, {'current': 4499.997, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.003, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 410.53357315063477}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2024-12-17 22:09:54,746 INFO    HandlerThread:82813 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-17 22:09:54,746 INFO    HandlerThread:82813 [system_monitor.py:probe():227] Publishing system info
+2024-12-17 22:09:54,746 DEBUG   HandlerThread:82813 [system_info.py:_save_code():44] Saving code
+2024-12-17 22:09:54,747 DEBUG   HandlerThread:82813 [system_info.py:_save_code():67] Saving code done
+2024-12-17 22:09:54,747 DEBUG   HandlerThread:82813 [system_info.py:_save_patches():84] Saving git patches
+2024-12-17 22:09:54,849 DEBUG   HandlerThread:82813 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-17 22:09:54,850 INFO    HandlerThread:82813 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-17 22:09:54,853 DEBUG   SenderThread:82813 [sender.py:send():379] send: files
+2024-12-17 22:09:54,853 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-17 22:09:54,853 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-17 22:09:54,854 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-17 22:09:55,082 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-17 22:09:55,082 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: python_packages
+2024-12-17 22:09:55,085 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:09:55,085 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:09:55,087 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:09:55,087 INFO    HandlerThread:82813 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/tb"
+  save: true
+}
+control {
+  relay_id: "fda16768da264ac7b1b49c3533ff68be"
+}
+_info {
+  stream_id: "7lp0oxke"
+}
+
+2024-12-17 22:09:55,087 DEBUG   HandlerThread:82813 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-17 22:09:55,106 INFO    wandb-upload_0:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/0gzqtvqw-wandb-metadata.json
+2024-12-17 22:09:55,119 INFO    wandb-upload_2:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/0iyvg9r1-diff.patch
+2024-12-17 22:09:55,156 INFO    wandb-upload_1:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/3inf5emz-code/FISH/eval_robot.py
+2024-12-17 22:09:55,215 DEBUG   SenderThread:82813 [sender.py:send():379] send: telemetry
+2024-12-17 22:09:55,215 DEBUG   SenderThread:82813 [sender.py:send():379] send: tbrecord
+2024-12-17 22:09:55,216 DEBUG   SenderThread:82813 [sender.py:send():379] send: files
+2024-12-17 22:09:55,216 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0 with policy live
+2024-12-17 22:09:55,413 INFO    wandb-upload_0:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/aljati01-events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0
+2024-12-17 22:09:55,636 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/requirements.txt
+2024-12-17 22:09:55,636 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-metadata.json
+2024-12-17 22:09:55,636 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/diff.patch
+2024-12-17 22:09:55,636 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0
+2024-12-17 22:09:55,636 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:09:55,636 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/code/FISH/eval_robot.py
+2024-12-17 22:09:55,636 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/code/FISH
+2024-12-17 22:09:55,636 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/code
+2024-12-17 22:09:56,082 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:09:57,083 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:09:57,635 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:09:58,083 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:09:59,083 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:00,082 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:10:00,083 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:00,475 DEBUG   SenderThread:82813 [sender.py:send():379] send: config
+2024-12-17 22:10:01,083 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:01,636 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:02,084 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:03,084 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:03,637 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:04,084 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:05,083 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:10:05,084 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:05,637 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:06,084 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:07,084 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:07,638 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:08,084 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:09,084 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:09,639 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:10,082 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:10:10,082 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:10:10,125 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:10,229 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:10:11,085 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:12,085 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:13,085 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:13,640 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:14,085 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:15,086 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:15,640 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:16,084 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:10:16,086 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:17,086 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:17,641 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:18,087 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:19,086 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:20,086 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:21,084 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:10:21,086 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:22,087 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:23,087 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:23,643 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:24,087 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:25,082 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:10:25,082 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:10:25,129 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:26,087 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:26,183 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:10:26,644 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/config.yaml
+2024-12-17 22:10:27,087 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:28,087 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:29,087 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:30,087 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:31,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:31,645 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:32,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:32,253 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:10:33,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:33,646 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:34,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:35,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:36,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:37,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:38,084 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:10:38,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:39,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:40,082 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:10:40,083 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:10:40,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:41,093 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:41,648 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:42,090 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:42,345 DEBUG   SenderThread:82813 [sender.py:send():379] send: files
+2024-12-17 22:10:42,345 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_f8fef5727eff9ff71143.mp4 with policy now
+2024-12-17 22:10:42,345 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 22:10:42,346 DEBUG   SenderThread:82813 [sender.py:send():379] send: history
+2024-12-17 22:10:42,346 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:10:42,347 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:10:42,597 INFO    wandb-upload_1:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/iz3o4t06-media/videos/eval/0_eval_0_f8fef5727eff9ff71143.mp4
+2024-12-17 22:10:42,648 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json
+2024-12-17 22:10:42,648 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/0_eval_0_f8fef5727eff9ff71143.mp4
+2024-12-17 22:10:42,649 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos
+2024-12-17 22:10:42,649 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media
+2024-12-17 22:10:42,649 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval
+2024-12-17 22:10:42,911 INFO    wandb-upload_0:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/e8hb5f9e-media/videos/eval/0_eval_0_f8fef5727eff9ff71143.mp4
+2024-12-17 22:10:43,090 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:43,349 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:10:43,649 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:44,090 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:45,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:46,091 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:47,090 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:47,650 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:48,091 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:48,458 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:10:49,091 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:49,651 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:50,091 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:51,091 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:52,091 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:52,652 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0
+2024-12-17 22:10:53,091 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:53,652 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:53,973 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:10:54,091 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:54,729 DEBUG   SystemMonitor:82813 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-17 22:10:54,729 DEBUG   SenderThread:82813 [sender.py:send():379] send: stats
+2024-12-17 22:10:55,083 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:10:55,083 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:10:55,125 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:55,653 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:56,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:57,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:57,653 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:58,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:59,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:59,868 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:11:00,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:01,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:02,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:03,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:04,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:05,085 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:11:05,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:06,093 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:07,093 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:08,093 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:09,093 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:10,083 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:11:10,083 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:11:10,129 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:10,240 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:11:11,093 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:12,093 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:13,093 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:13,659 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:14,094 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:15,094 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:15,569 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:11:15,659 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:16,096 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:17,095 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:17,660 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:18,097 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:19,096 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:19,661 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:20,096 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:21,085 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:11:21,096 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:21,662 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:22,096 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:23,096 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:23,662 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:23,747 DEBUG   SenderThread:82813 [sender.py:send():379] send: files
+2024-12-17 22:11:23,747 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_8a67326b5dfefa361130.mp4 with policy now
+2024-12-17 22:11:23,750 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 22:11:23,751 DEBUG   SenderThread:82813 [sender.py:send():379] send: history
+2024-12-17 22:11:23,751 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:11:23,751 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:11:24,004 INFO    wandb-upload_1:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/wz2w0qdj-media/videos/eval/1_eval_1_8a67326b5dfefa361130.mp4
+2024-12-17 22:11:24,097 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:24,663 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json
+2024-12-17 22:11:24,663 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/1_eval_1_8a67326b5dfefa361130.mp4
+2024-12-17 22:11:24,664 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval
+2024-12-17 22:11:24,731 DEBUG   SenderThread:82813 [sender.py:send():379] send: stats
+2024-12-17 22:11:24,963 INFO    wandb-upload_0:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/tplhjs43-media/videos/eval/1_eval_1_8a67326b5dfefa361130.mp4
+2024-12-17 22:11:25,085 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:11:25,085 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:11:25,129 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:25,663 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:26,101 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:26,187 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:11:26,664 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:27,102 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:28,100 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:29,105 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:29,664 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:30,102 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:31,114 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:31,665 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:32,086 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:11:32,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:33,103 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:34,105 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:35,104 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:35,666 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0
+2024-12-17 22:11:35,910 INFO    wandb-upload_1:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/f5kpna1a-events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0
+2024-12-17 22:11:36,103 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:36,489 DEBUG   SenderThread:82813 [sender.py:send():379] send: history
+2024-12-17 22:11:36,489 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:11:36,489 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:11:36,667 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json
+2024-12-17 22:11:37,110 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:37,490 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:11:37,667 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:38,110 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:39,107 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:40,096 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:11:40,096 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:11:40,137 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:41,107 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:41,668 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:42,108 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:43,108 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:43,385 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:11:43,669 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:44,108 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:45,108 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:46,108 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:47,108 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:47,671 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:48,108 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:49,108 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:49,329 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:11:49,671 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:50,108 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:51,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:52,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:53,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:53,673 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:54,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:54,730 DEBUG   SenderThread:82813 [sender.py:send():379] send: stats
+2024-12-17 22:11:54,731 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:11:55,086 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:11:55,087 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:11:55,129 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:55,673 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:56,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:57,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:58,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:59,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:00,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:00,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:01,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:02,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:03,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:04,110 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:05,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:05,110 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:06,110 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:07,110 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:08,110 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:08,911 DEBUG   SenderThread:82813 [sender.py:send():379] send: files
+2024-12-17 22:12:08,911 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_4735c6188bf9f36869cb.mp4 with policy now
+2024-12-17 22:12:08,914 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 22:12:08,914 DEBUG   SenderThread:82813 [sender.py:send():379] send: history
+2024-12-17 22:12:08,914 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:12:08,914 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:12:09,118 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:09,174 INFO    wandb-upload_0:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/b0oeewff-media/videos/eval/2_eval_2_4735c6188bf9f36869cb.mp4
+2024-12-17 22:12:09,678 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:12:09,678 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json
+2024-12-17 22:12:09,678 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/2_eval_2_4735c6188bf9f36869cb.mp4
+2024-12-17 22:12:09,678 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval
+2024-12-17 22:12:10,003 INFO    wandb-upload_1:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/w5y0lzh2-media/videos/eval/2_eval_2_4735c6188bf9f36869cb.mp4
+2024-12-17 22:12:10,090 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:12:10,091 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:12:10,133 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:10,234 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:11,114 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:11,678 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:12:12,123 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:13,120 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:13,679 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:12:14,133 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:15,125 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:15,339 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:16,131 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:16,680 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0
+2024-12-17 22:12:17,134 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:17,324 DEBUG   SenderThread:82813 [sender.py:send():379] send: history
+2024-12-17 22:12:17,324 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:12:17,325 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:12:17,681 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:12:17,681 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json
+2024-12-17 22:12:18,135 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:19,145 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:19,681 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:12:20,141 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:20,399 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:21,142 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:21,682 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:12:22,137 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:23,138 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:23,683 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:12:24,138 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:24,731 DEBUG   SenderThread:82813 [sender.py:send():379] send: stats
+2024-12-17 22:12:25,090 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:12:25,090 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:12:25,138 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:26,139 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:26,223 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:27,140 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:28,141 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:29,141 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:30,142 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:31,141 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:32,093 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:32,142 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:33,144 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:34,143 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:35,141 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:36,142 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:37,093 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:37,143 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:38,143 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:39,144 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:40,089 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:12:40,089 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:12:40,144 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:41,143 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:42,145 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:42,184 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:43,145 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:44,144 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:45,144 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:46,144 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:47,144 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:48,094 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:48,145 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:49,145 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:50,173 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:50,390 DEBUG   SenderThread:82813 [sender.py:send():379] send: files
+2024-12-17 22:12:50,390 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_a4af24fd7d85bfc8c826.mp4 with policy now
+2024-12-17 22:12:50,394 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 22:12:50,395 DEBUG   SenderThread:82813 [sender.py:send():379] send: history
+2024-12-17 22:12:50,395 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:12:50,396 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:12:50,642 INFO    wandb-upload_0:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/o2w5qpu6-media/videos/eval/3_eval_3_a4af24fd7d85bfc8c826.mp4
+2024-12-17 22:12:50,691 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json
+2024-12-17 22:12:50,691 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/3_eval_3_a4af24fd7d85bfc8c826.mp4
+2024-12-17 22:12:50,692 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval
+2024-12-17 22:12:50,960 INFO    wandb-upload_1:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/69ml2jnp-media/videos/eval/3_eval_3_a4af24fd7d85bfc8c826.mp4
+2024-12-17 22:12:51,162 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:51,692 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:12:52,156 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:53,153 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:53,406 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:54,158 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:54,731 DEBUG   SenderThread:82813 [sender.py:send():379] send: stats
+2024-12-17 22:12:55,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:12:55,092 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:12:55,159 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:55,693 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:12:56,162 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:57,152 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:57,694 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:12:58,156 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:58,873 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:59,163 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:00,165 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:01,162 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:02,159 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:03,169 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:04,104 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:13:04,167 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:05,163 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:06,167 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:07,173 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:08,174 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:09,175 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:09,384 DEBUG   SenderThread:82813 [sender.py:send():379] send: history
+2024-12-17 22:13:09,384 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:13:09,384 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:13:09,384 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:13:09,697 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:09,698 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json
+2024-12-17 22:13:09,698 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0
+2024-12-17 22:13:09,954 INFO    wandb-upload_0:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/5zmrcuk8-events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0
+2024-12-17 22:13:10,097 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:13:10,098 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:13:10,176 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:11,181 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:11,698 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:12,180 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:13,177 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:13,699 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:14,178 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:14,672 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:13:15,178 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:15,700 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:16,178 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:17,178 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:18,178 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:19,178 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:20,098 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:13:20,178 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:21,178 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:22,179 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:23,179 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:24,179 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:24,732 DEBUG   SenderThread:82813 [sender.py:send():379] send: stats
+2024-12-17 22:13:25,089 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:13:25,090 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:13:25,179 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:25,221 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:13:25,703 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:26,179 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:27,179 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:28,180 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:29,180 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:30,180 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:31,099 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:13:31,180 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:32,180 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:33,180 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:34,180 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:35,181 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:36,181 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:37,099 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:13:37,183 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:38,181 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:39,184 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:40,090 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:13:40,090 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:13:40,183 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:41,195 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:41,238 DEBUG   SenderThread:82813 [sender.py:send():379] send: files
+2024-12-17 22:13:41,238 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_ccefe4eb2a70fd4856d6.mp4 with policy now
+2024-12-17 22:13:41,285 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 22:13:41,285 DEBUG   SenderThread:82813 [sender.py:send():379] send: history
+2024-12-17 22:13:41,286 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:13:41,286 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:13:41,498 INFO    wandb-upload_1:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/chvo9qog-media/videos/eval/4_eval_4_ccefe4eb2a70fd4856d6.mp4
+2024-12-17 22:13:41,708 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:41,708 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json
+2024-12-17 22:13:41,709 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/4_eval_4_ccefe4eb2a70fd4856d6.mp4
+2024-12-17 22:13:41,709 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval
+2024-12-17 22:13:41,976 INFO    wandb-upload_0:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/b9kgywxw-media/videos/eval/4_eval_4_ccefe4eb2a70fd4856d6.mp4
+2024-12-17 22:13:42,187 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:42,286 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:13:43,186 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:43,709 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:44,195 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:45,198 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:45,710 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:46,195 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:47,198 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:47,710 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:47,830 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:13:48,207 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:49,219 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:49,711 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:49,957 DEBUG   SenderThread:82813 [sender.py:send():379] send: exit
+2024-12-17 22:13:49,957 INFO    SenderThread:82813 [sender.py:send_exit():586] handling exit code: 0
+2024-12-17 22:13:49,957 INFO    SenderThread:82813 [sender.py:send_exit():588] handling runtime: 235
+2024-12-17 22:13:49,958 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:13:49,958 INFO    SenderThread:82813 [sender.py:send_exit():594] send defer
+2024-12-17 22:13:49,958 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:13:49,958 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-17 22:13:49,958 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:13:49,958 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-17 22:13:49,958 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 1
+2024-12-17 22:13:49,958 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:13:49,958 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-17 22:13:49,959 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:13:49,959 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-17 22:13:49,959 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 2
+2024-12-17 22:13:49,959 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:13:49,959 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-17 22:13:49,959 INFO    HandlerThread:82813 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-17 22:13:49,959 DEBUG   SystemMonitor:82813 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-17 22:13:49,959 INFO    HandlerThread:82813 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-17 22:13:49,959 DEBUG   SystemMonitor:82813 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-17 22:13:49,959 INFO    HandlerThread:82813 [interfaces.py:finish():200] Joined disk monitor
+2024-12-17 22:13:50,007 INFO    HandlerThread:82813 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-17 22:13:50,007 INFO    HandlerThread:82813 [interfaces.py:finish():200] Joined memory monitor
+2024-12-17 22:13:50,007 INFO    HandlerThread:82813 [interfaces.py:finish():200] Joined network monitor
+2024-12-17 22:13:50,008 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:13:50,008 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-17 22:13:50,008 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 3
+2024-12-17 22:13:50,008 DEBUG   SenderThread:82813 [sender.py:send():379] send: stats
+2024-12-17 22:13:50,008 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:13:50,008 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-17 22:13:50,008 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:13:50,008 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-17 22:13:50,008 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 4
+2024-12-17 22:13:50,008 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:13:50,008 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-17 22:13:50,711 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json
+2024-12-17 22:13:50,712 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0
+2024-12-17 22:13:51,712 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:56,010 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:13:56,011 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:13:56,011 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:13:56,011 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-17 22:13:56,011 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-17 22:13:56,011 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 5
+2024-12-17 22:13:56,011 DEBUG   SenderThread:82813 [sender.py:send():379] send: history
+2024-12-17 22:13:56,011 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:13:56,011 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:13:56,012 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-17 22:13:56,013 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:13:56,013 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:13:56,013 DEBUG   SenderThread:82813 [sender.py:send():379] send: history
+2024-12-17 22:13:56,013 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:13:56,017 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:13:56,017 DEBUG   SenderThread:82813 [sender.py:send():379] send: summary
+2024-12-17 22:13:56,017 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:13:56,017 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:13:56,017 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-17 22:13:56,018 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 6
+2024-12-17 22:13:56,018 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:13:56,018 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-17 22:13:56,018 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:13:56,018 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-17 22:13:56,018 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 7
+2024-12-17 22:13:56,018 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:13:56,018 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:13:56,018 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-17 22:13:56,018 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:13:56,018 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-17 22:13:56,713 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json
+2024-12-17 22:13:56,958 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:13:57,893 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 8
+2024-12-17 22:13:57,897 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:13:57,897 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:13:57,898 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-17 22:13:57,898 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:13:57,898 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-17 22:13:57,898 INFO    SenderThread:82813 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-17 22:13:57,900 INFO    SenderThread:82813 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-17 22:13:57,914 INFO    SenderThread:82813 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-17 22:13:57,924 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 9
+2024-12-17 22:13:57,924 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:13:57,924 DEBUG   SenderThread:82813 [sender.py:send():379] send: artifact
+2024-12-17 22:13:57,925 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-17 22:13:57,959 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:13:58,703 INFO    wandb-upload_0:82813 [upload_job.py:push():85] Skipped uploading /tmp/tmpzprnyq1w/wandb-job.json
+2024-12-17 22:13:58,703 INFO    wandb-upload_2:82813 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp9f2r2ubh
+2024-12-17 22:13:58,714 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:58,826 INFO    wandb-upload_1:82813 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmptk53d0ug
+2024-12-17 22:13:59,393 INFO    SenderThread:82813 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM4MzM4ODU2OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM4MzM3MTUyNg==', 'versionIndex': 382}}}
+2024-12-17 22:13:59,393 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:13:59,393 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-17 22:13:59,393 INFO    SenderThread:82813 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-17 22:13:59,715 INFO    SenderThread:82813 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files
+2024-12-17 22:13:59,715 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/config.yaml config.yaml
+2024-12-17 22:13:59,715 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json wandb-summary.json
+2024-12-17 22:13:59,715 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/requirements.txt requirements.txt
+2024-12-17 22:13:59,718 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-metadata.json wandb-metadata.json
+2024-12-17 22:13:59,718 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0 events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0
+2024-12-17 22:13:59,719 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log output.log
+2024-12-17 22:13:59,720 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/diff.patch diff.patch
+2024-12-17 22:13:59,720 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-17 22:13:59,720 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/1_eval_1_8a67326b5dfefa361130.mp4 media/videos/eval/1_eval_1_8a67326b5dfefa361130.mp4
+2024-12-17 22:13:59,720 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/0_eval_0_f8fef5727eff9ff71143.mp4 media/videos/eval/0_eval_0_f8fef5727eff9ff71143.mp4
+2024-12-17 22:13:59,720 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/4_eval_4_ccefe4eb2a70fd4856d6.mp4 media/videos/eval/4_eval_4_ccefe4eb2a70fd4856d6.mp4
+2024-12-17 22:13:59,720 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/3_eval_3_a4af24fd7d85bfc8c826.mp4 media/videos/eval/3_eval_3_a4af24fd7d85bfc8c826.mp4
+2024-12-17 22:13:59,722 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/2_eval_2_4735c6188bf9f36869cb.mp4 media/videos/eval/2_eval_2_4735c6188bf9f36869cb.mp4
+2024-12-17 22:13:59,722 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 10
+2024-12-17 22:13:59,724 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:13:59,724 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:13:59,724 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-17 22:13:59,724 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:13:59,724 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-17 22:13:59,724 INFO    SenderThread:82813 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 22:13:59,900 INFO    wandb-upload_1:82813 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/requirements.txt
+2024-12-17 22:13:59,920 INFO    wandb-upload_0:82813 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/config.yaml
+2024-12-17 22:13:59,932 INFO    wandb-upload_2:82813 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json
+2024-12-17 22:13:59,943 INFO    wandb-upload_4:82813 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:59,960 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:13:59,960 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:14:00,035 INFO    wandb-upload_3:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/gqgju0j7-events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0
+2024-12-17 22:14:00,236 INFO    Thread-11 (_thread_body):82813 [sender.py:transition_state():614] send defer: 11
+2024-12-17 22:14:00,236 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:14:00,236 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-17 22:14:00,236 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:14:00,236 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-17 22:14:00,236 INFO    SenderThread:82813 [file_pusher.py:join():175] waiting for file pusher
+2024-12-17 22:14:00,237 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 12
+2024-12-17 22:14:00,237 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:14:00,237 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-17 22:14:00,237 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:14:00,237 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-17 22:14:00,238 INFO    SenderThread:82813 [file_stream.py:finish():601] file stream finish called
+2024-12-17 22:14:00,435 INFO    SenderThread:82813 [file_stream.py:finish():605] file stream finish is done
+2024-12-17 22:14:00,436 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 13
+2024-12-17 22:14:00,436 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:14:00,436 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-17 22:14:00,436 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:14:00,436 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-17 22:14:00,436 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 14
+2024-12-17 22:14:00,436 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:14:00,436 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-17 22:14:00,437 DEBUG   SenderThread:82813 [sender.py:send():379] send: final
+2024-12-17 22:14:00,437 DEBUG   SenderThread:82813 [sender.py:send():379] send: footer
+2024-12-17 22:14:00,437 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:14:00,437 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-17 22:14:00,437 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:14:00,437 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:14:00,438 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: server_info
+2024-12-17 22:14:00,438 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: server_info
+2024-12-17 22:14:00,438 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:14:00,440 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-17 22:14:00,440 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-17 22:14:00,440 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:14:00,521 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:14:00,521 INFO    MainThread:82813 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-17 22:14:00,522 INFO    MainThread:82813 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-17 22:14:00,522 INFO    MainThread:82813 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-17 22:14:00,522 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-17 22:14:00,522 INFO    HandlerThread:82813 [handler.py:finish():882] shutting down handler
+2024-12-17 22:14:01,440 INFO    WriterThread:82813 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/run-7lp0oxke.wandb
+2024-12-17 22:14:01,521 INFO    SenderThread:82813 [sender.py:finish():1615] shutting down sender
+2024-12-17 22:14:01,521 INFO    SenderThread:82813 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 22:14:01,521 INFO    SenderThread:82813 [file_pusher.py:join():175] waiting for file pusher

220948/wandb/debug.log

@@ -0,0 +1,31 @@
+2024-12-17 22:09:54,388 INFO    MainThread:82633 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-17 22:09:54,388 INFO    MainThread:82633 [wandb_setup.py:_flush():76] Configure stats pid to 82633
+2024-12-17 22:09:54,388 INFO    MainThread:82633 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-17 22:09:54,388 INFO    MainThread:82633 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/settings
+2024-12-17 22:09:54,388 INFO    MainThread:82633 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-17 22:09:54,388 INFO    MainThread:82633 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-17 22:09:54,388 INFO    MainThread:82633 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-17 22:09:54,389 INFO    MainThread:82633 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-17 22:09:54,389 INFO    MainThread:82633 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/logs/debug.log
+2024-12-17 22:09:54,389 INFO    MainThread:82633 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/logs/debug-internal.log
+2024-12-17 22:09:54,389 INFO    MainThread:82633 [wandb_init.py:init():569] calling init triggers
+2024-12-17 22:09:54,389 INFO    MainThread:82633 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '510_0', 'wandb_notes': '510_0_', 'eval': True, 'true_action_history': False, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': False, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['modelsys_twodim'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/510_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948'}
+2024-12-17 22:09:54,389 INFO    MainThread:82633 [wandb_init.py:init():619] starting backend
+2024-12-17 22:09:54,389 INFO    MainThread:82633 [wandb_init.py:init():623] setting up manager
+2024-12-17 22:09:54,392 INFO    MainThread:82633 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-17 22:09:54,394 INFO    MainThread:82633 [wandb_init.py:init():631] backend started and connected
+2024-12-17 22:09:54,409 INFO    MainThread:82633 [wandb_init.py:init():720] updated telemetry
+2024-12-17 22:09:54,416 INFO    MainThread:82633 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-17 22:09:54,649 INFO    MainThread:82633 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-17 22:09:54,716 INFO    MainThread:82633 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-17 22:09:54,716 INFO    MainThread:82633 [wandb_init.py:init():804] starting run threads in backend
+2024-12-17 22:09:55,083 INFO    MainThread:82633 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-17 22:09:55,083 INFO    MainThread:82633 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-17 22:09:55,083 INFO    MainThread:82633 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-17 22:09:55,083 INFO    MainThread:82633 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-17 22:09:55,086 INFO    MainThread:82633 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-17 22:09:55,086 INFO    MainThread:82633 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/tb, True
+2024-12-17 22:10:00,474 INFO    MainThread:82633 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'}
+2024-12-17 22:14:01,524 WARNING MsgRouterThr:82633 [router.py:message_loop():77] message_loop has been closed

220948/wandb/run-20241217_220954-7lp0oxke/files/code/FISH/eval_robot.py

@@ -0,0 +1,512 @@
+#%%
+import warnings
+import os
+
+os.environ['MKL_SERVICE_FORCE_INTEL'] = '1'
+os.environ['MUJOCO_GL'] = 'egl'
+from pathlib import Path
+#%%
+import hydra
+import numpy as np
+import torch
+
+import utils
+from utils import get_feature_dirname_from_configs
+
+from video import VideoRecorder
+import pickle
+import time
+import threading
+import shutil
+from logger import Logger
+
+import wandb 
+from omegaconf import OmegaConf, open_dict
+
+from replay_buffer_robot import RosbagEvalReplayBufferStorage
+from lerobot.common.utils.utils import _relative_path_between
+
+torch.backends.cudnn.benchmark = True
+warnings.filterwarnings('ignore', category=DeprecationWarning)
+
+# import specs for replay buffer
+from dm_env import specs
+
+import sys, signal
+import yaml
+
+# get path of current file
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(current_path, os.pardir))
+# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list
+
+def make_agent(obs_spec, action_spec, cfg):
+	cfg.obs_shape = obs_spec['pixels'].shape
+	dataset_statistics = None # this will be loaded from the checkpoint
+	try:
+		cfg.action_shape = action_spec.shape
+	except:
+		pass
+	return hydra.utils.instantiate(cfg, dataset_statistics)
+
+class Workspace:
+	def __init__(self, cfg):
+		self.work_dir = Path.cwd()
+		print(f'workspace: {self.work_dir}')
+		
+		signal.signal(signal.SIGINT, self.signal_handler)
+		
+		self.cfg = cfg
+		self.loading_uncompiled_checkpoint_with_compile = False
+		self.loading_compiled_checkpoint_with_no_compile = False
+
+		snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' 
+		self.load_checkpoint_conf(snapshot_path=snapshot_path)
+
+		# load config for action trajectories
+		utils.set_seed_everywhere(self.cfg.seed)
+		self.device = torch.device(self.cfg.device)
+		self.setup()
+
+		# self.agent = make_agent(self.eval_env.observation_spec(),
+		# 						self.eval_env.action_spec(), self.cfg.agent)
+		self.timer = utils.Timer()
+		# self._global_step = 0
+		self._global_episode = 0
+		self._global_epoch = 0
+		self.num_episode_successes = 0
+
+		# Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra
+		with open_dict(self.cfg):
+			self.cfg.feature_type = get_feature_dirname_from_configs(
+				hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), 
+				self.cfg.agent.config.policy_cfg.input_shapes,
+				hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None,
+			)
+
+		wandb_config = OmegaConf.to_container(
+			self.cfg, resolve=True, throw_on_missing=True
+		)
+		# must be called before any tf summary writer is created
+		if self.cfg.use_wandb:
+			wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config)
+
+		self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb)
+
+		# if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile:
+		# 	self.agent.compile_modules()
+			
+		# self.load_checkpoint(snapshot_path=snapshot_path)
+
+		# if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint
+		# 	self.agent.compile_modules()
+
+		print(f"loaded agent with feature_type: {self.cfg.feature_type}")
+
+	def check_for_key_press(self):
+		while self.continue_keypress_thread:
+			inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n")
+			if inp == 'n':
+				self.preempt_episode = True
+				print("preempting episode")
+			elif inp in ['', '0', '1']: # enter key
+				if inp in ['0', '1']:
+					self.num_episode_successes += int(inp)
+				self.proceed_after_env_reset_event.set()
+				print("proceeding to start episode!")
+			elif inp == 'q':
+				self.proceed_after_env_reset_event.set()
+				self.preempt_episode = True
+				self.exit_eval = True
+				self.continue_keypress_thread = False # will stop the keypress thread
+				print("quitting eval")
+				break
+			elif inp == 'r':
+				print('restarting episode')
+				self.preempt_episode = True
+				self.restart_episode = True
+			else:
+				print("Invalid key press, try again")
+		
+		# self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def signal_handler(self, signal, frame):
+		print("\nprogram exiting gracefully")
+		self.proceed_after_env_reset_event.set()
+		self.preempt_episode = True
+		self.exit_eval = True
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+		video_filepath = self.video_recorder.save()
+		# get the video file and convert to video tensor to log 
+		self.logger.log_video('eval/video', video_filepath, self.global_step)
+		sys.exit(0)
+		
+	def setup(self):
+		# create envs
+		self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn)
+		# expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml')
+		# self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader)
+		# self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm']
+		# self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm']
+		# self.eval_env._env.action_period = expert_demo_config['sample_period']
+		# print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}")
+		# print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}")
+
+		# self.eval_env.set_demo_params(self.cfg.expert_dataset)
+
+		# Turn off random start
+		self.eval_env.random_start = False
+
+		# create replay buffer
+		# data_specs = [
+		# 	{
+		# 		'observation': self.eval_env.observation_spec(),
+		# 	},
+		# 	# self.eval_env.observation_spec()['features'],
+		# 	self.eval_env.action_spec(),
+		# 	specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), 
+		# 	specs.Array((1, ), np.float32, 'reward'),
+		# 	specs.Array((1, ), np.float32, 'discount'),
+		# ]
+
+		# self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict())
+		self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir)
+
+		self.video_recorder = VideoRecorder(
+			self.work_dir if self.cfg.save_video else None,
+			ros_enabled=True,
+			fps=self.cfg.agent.config.policy_frequency,
+			)
+		
+		print('workspace setup complete')
+		
+	@property
+	def global_step(self):
+		# return self._global_step
+		return self.eval_env.get_global_step()
+
+	@property
+	def global_episode(self):
+		return self._global_episode
+
+	@property
+	def global_frame(self):
+		return self.global_step * self.cfg.action_repeat
+
+	@property
+	def global_epoch(self):
+		return self._global_epoch
+	
+	def reset(self, eval_idx):
+		if not self.eval_env.enable_arm:
+			return np.array([0,0,0], dtype=np.float32)
+		self.eval_env.arm_refresh(reset=False)
+		# Set start position
+		try:
+			self.eval_env.set_position(self.start_pos[eval_idx])
+		except:
+			self.eval_env.arm.set_position(self.start_pos[eval_idx])
+			if self.eval_env.arm.keep_gripper_closed:
+				self.eval_env.arm.close_gripper_fully()
+			else:
+				self.eval_env.arm.open_gripper_fully()
+		time.sleep(0.1)		
+		time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32),
+									   np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32))	
+		return time_step
+
+	def eval(self):
+		# before evals start, prompt user for name of grasped object and the left book of the slot location
+		grasped_obj_name = input("Enter the name of the grasped object: ")
+		left_book_slot = input("Enter the left book slot location: ")
+		# update wandb config
+		if self.cfg.use_wandb:
+			wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot})
+
+		self.preempt_episode = False
+		self.exit_eval = False
+		self.restart_episode = False
+
+		self.continue_keypress_thread = True
+		self.proceed_after_env_reset_event = threading.Event()
+		self.keypress_input_thread = threading.Thread(target=self.check_for_key_press)
+		self.keypress_input_thread.start()
+
+		# # Set model to eval mode
+		# self.agent.train(False)
+		
+		eval_until_episode = utils.Until(self.cfg.num_eval)
+
+		self.use_action_history = False
+		# if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+		if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes:
+			self.use_action_history = True
+
+		# self.eval_replay_storage._new_eval_step(0)
+
+		# if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent):
+		# 	with open(self.cfg.expert_dataset, 'rb') as f:
+		# 		if self.cfg.obs_type == 'pixels':
+		# 			self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f)
+		# 		elif self.cfg.obs_type == 'features':
+		# 			_, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f)
+				
+		# 		if self.cfg.action_trajectories:
+		# 			with open(self.cfg.expert_action_trajectories, 'rb') as f:
+		# 				self.expert_action = pickle.load(f)
+
+		# 		if isinstance(self.cfg.train_demo_idxs_list_or_num, int):
+		# 			if self.cfg.train_demo_idxs_list_or_num == -1:
+		# 				self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo)
+		# 			train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num))
+				
+		# 		self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num]
+		# 		self.expert_action = self.expert_action[train_demo_idxs_list_or_num]
+		# 		self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num]
+		# 		# if self.cfg.action_plans:
+		# 		# 	self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num]
+		# 		# self.expert_demo = self.expert_demo[:self.cfg.num_demos]
+		# 		# self.expert_action = self.expert_action[:self.cfg.num_demos]
+		# 		# self.expert_reward = self.expert_reward[:self.cfg.num_demos]
+
+		# 		self.expert_demo = np.concatenate(self.expert_demo, axis=0)
+		# 		self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8))
+		# 		self.expert_action = np.concatenate(self.expert_action, axis=0)
+				
+		# 		self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config)
+		
+		# Get start points
+		if self.cfg.random_start:
+			eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl'
+			if eval_starts.exists():
+				with eval_starts.open('rb') as f:
+					self.start_pos = pickle.load(f)
+			else:
+				eval_starts = Path(self.cfg.eval_starts)
+				eval_starts.mkdir(parents=True, exist_ok=True)
+				
+				# Generate start points
+				self.start_pos = []
+				try:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.get_random_pos())
+				except:
+					for _ in range(self.cfg.num_eval):
+						self.start_pos.append(self.eval_env.arm.get_random_pos())
+
+				# Save start points for the task
+				eval_starts = eval_starts / 'starts.pkl'
+				with eval_starts.open('wb') as f:
+					pickle.dump(self.start_pos, f)
+
+		time_step = self.eval_env.reset()
+		# replay_thread = None
+		while eval_until_episode(self.global_episode) and not self.exit_eval:
+			# self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			print(f"Starting episode {self.global_episode}")
+			time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped
+			self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4')
+			# x = input("Press Enter to continue... after reseting env")
+			print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success")
+			self.proceed_after_env_reset_event.clear() # clear the event flag
+			self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+			if self.global_episode > 0:
+				self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+				self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			time_step = self.eval_env.reset()
+			# debug_info_dict = self.eval_env.debug_info_dict
+			# if replay_thread is not None:
+			# 	# wait for the last replay thread to finish
+			# 	replay_thread.join()
+
+			# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+			# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+
+			# replay_thread.start()
+			if self.cfg.random_start:
+				time_step = self.reset(self.global_episode)
+				time.sleep(2) #5)
+			# if 'vinn' in repr(self.agent):
+			# 	self.agent.reset()
+			# 	# self.agent.buffer.reset()
+			# 	# if self.cfg.open_loop:
+			# 	# 	self.agent.current_step = 0
+			# if 'openloop' in repr(self.agent):
+			# 	self.agent.curr_step = 0
+			# at start of each episode, provide zero action for policies that use action history
+			# shape should be (T_o, T_a, action_dim)
+
+			# while not time_step.last() and not self.preempt_episode:
+			self.video_recorder.ros_start_recording()
+			self.eval_replay_storage.start_episode()
+			self.eval_env.start_policy_timer()
+			while not self.eval_env.episode_done() and not self.preempt_episode:
+				# with torch.no_grad(), utils.eval_mode(self.agent):
+				# 	# if self.cfg.agent.provide_topk:
+				# 	# 	action, vinn_action, topk = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# elif self.cfg.agent.provide_obs:
+				# 	# 	action, vinn_action, obs = self.agent.act(
+				# 	# 		time_step.observation['pixels'],
+				# 	# 		self.global_step,
+				# 	# 		eval_mode=True)
+				# 	# else:
+				# 	action, vinn_action = self.agent.act(
+				# 		time_step.observation,
+				# 		self.global_step,
+				# 		eval_mode=True,
+				# 		obs_timestamp=time_step.observation['timestamp'],
+				# 		obs_seq=time_step.observation['seq'],
+				# 		action_history=action_history,
+				# 		action_history_start_timestamp=action_history_start_timestamp,
+				# 		)
+					# DONT WAIT FOR POLICY TO GET AN ACTION 
+					# we dont want to slow down grabbing obs and passing to sam/contact features
+
+				self.eval_env.run_policy_threads() # this just does a rospy sleep
+					
+				# if self.use_action_history:
+				# 	action_history_start_timestamp = time_step.observation['timestamp']
+				# 	# action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...]		
+				# 	# add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1
+				# 	# TODO: handle n_obs_steps > 1
+				# 	action_history = action[np.newaxis, ...]
+
+				# time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken
+				# debug_info_dict = self.eval_env.debug_info_dict
+				
+				# time_step = self.eval_env.ros_step()
+				
+				# replay_thread.join()
+
+				# time how long it takes to execute the step
+				# time_before_add = time.perf_counter()
+				# self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)
+				# use thread to call the add function in a separate thread
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict))
+
+				# replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict))
+				# replay_thread.start()
+				
+				# print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}")
+
+				# self.video_recorder.record(self.eval_env)
+				# self._global_step += 1
+			
+			self.eval_env.stop_policy_timer()
+			
+			if self.restart_episode:
+				# means we should delete the current episode and start again
+				self.restart_episode = False
+				self.eval_replay_storage.reset_current_episode()
+				self.video_recorder.reset_current_episode()
+
+			else:
+				self.eval_replay_storage.store_current_episode()
+				video_filepath = self.video_recorder.save()
+				self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step)
+				self._global_episode += 1
+			
+			self.preempt_episode = False # reset preempt_episode flag
+		
+		# self.video_recorder.save(f'{episode}_eval.mp4')
+		# get the video file and convert to video tensor to log 
+
+		self.eval_env.reset()
+		
+		print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success")
+		self.proceed_after_env_reset_event.clear() # clear the event flag
+		self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set
+		if self.global_episode > 0:
+			# self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode)
+			self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode)
+			
+		self.continue_keypress_thread = False # will stop the keypress thread
+		self.keypress_input_thread.join() # wait for the keypress thread to finish
+
+	def load_checkpoint_conf(self, snapshot_path):
+		config_path = snapshot_path.parent / 'config.yaml'
+		if not config_path.exists():
+			raise FileNotFoundError(f'No snapshot conf found at {config_path}')
+		else:
+			# load the omegaconf config
+			hydra.core.global_hydra.GlobalHydra.instance().clear()
+			hydra.initialize(
+				str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)),
+			)
+			cfg = hydra.compose(Path(config_path).stem)
+			from deepdiff import DeepDiff
+			from omegaconf import open_dict
+			diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new
+			# import re
+			overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]]
+			if "values_changed" in diff:
+				# top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion
+				# for keys above, overwrite the old config with the new config
+				for k, v in diff['values_changed'].items():
+					# replace any keys that are under "root['suite']"
+					if k in overwriteable_keys or k.startswith("root['suite']"):
+						print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config")
+						if k == "root['agent']['config']['compile']":
+							if diff['values_changed'][k]['new_value']:
+								self.loading_uncompiled_checkpoint_with_compile = True
+							elif not diff['values_changed'][k]['new_value']:
+								# raise ValueError("Cannot load a compiled checkpoint without compile")
+								self.loading_compiled_checkpoint_with_no_compile = True
+						exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}")
+			# for any new values, update the old checkpoint config
+			if "dictionary_item_added" in diff:
+				for new_key in diff['dictionary_item_added']: # this is a list
+					# if new_key == "root['suite']['task_make_fn']['observation_cfg']":
+					if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']":
+						# pass the agents observation_cfg to the suite task_make_fn
+						with open_dict(cfg): # to allow addition of non-existing keys
+							# cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg
+							cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config
+						continue
+					elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key:
+						# skip adding the new key if it is the input_shapes of the policy_cfg
+						continue
+					else:
+						print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config")
+					# eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg'))
+					if new_key == "root['agent']['config']['compile']":
+						if self.cfg.agent.config.compile:
+							self.loading_uncompiled_checkpoint_with_compile = True
+					
+					with open_dict(cfg):
+						exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}")
+			self.cfg = cfg
+
+	def load_checkpoint(self, snapshot_path, bc=False):
+		print(f'resuming {repr(self.agent)}: {snapshot_path}')
+		with snapshot_path.open('rb') as f:
+			payload = torch.load(f)
+		agent_payload = {}
+		for k, v in payload.items():
+			if k not in self.__dict__:
+				agent_payload[k] = v
+			elif k == '_global_epoch':
+				self._global_epoch = v
+				print(f'loaded epoch: {v}')
+				if self.cfg.use_wandb:
+					# add to config of wandb
+					wandb.config.update({'epoch': v})
+
+		# self.agent.load_snapshot_eval(agent_payload, bc)
+
+@hydra.main(config_path='cfgs', config_name='config_eval')
+def main(cfg):
+	from eval_robot import Workspace as W
+	root_dir = Path.cwd()
+	workspace = W(cfg)
+
+	workspace.eval()
+
+if __name__ == '__main__':
+	main()

220948/wandb/run-20241217_220954-7lp0oxke/files/config.yaml

@@ -0,0 +1,891 @@
+wandb_version: 1
+
+root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon
+replay_buffer_size:
+  desc: null
+  value: 150000
+replay_buffer_num_workers:
+  desc: null
+  value: 2
+nstep:
+  desc: null
+  value: 3
+batch_size:
+  desc: null
+  value: 128
+seed:
+  desc: null
+  value: 0
+dataset_shuffle_seed:
+  desc: null
+  value: 0
+device:
+  desc: null
+  value: cuda
+save_video:
+  desc: null
+  value: true
+save_train_video:
+  desc: null
+  value: true
+use_tb:
+  desc: null
+  value: true
+use_wandb:
+  desc: null
+  value: true
+wandb_run_id:
+  desc: null
+  value: '510_0'
+wandb_notes:
+  desc: null
+  value: '510_0_'
+eval:
+  desc: null
+  value: true
+true_action_history:
+  desc: null
+  value: false
+process_contact_features:
+  desc: null
+  value: true
+obs_type:
+  desc: null
+  value: pixels
+use_color:
+  desc: null
+  value: true
+use_depth:
+  desc: null
+  value: true
+use_masks:
+  desc: null
+  value: false
+mask_list:
+  desc: null
+  value:
+  - EE_obj_mask
+mask_representation:
+  desc: null
+  value: channels
+crop_hw:
+  desc: null
+  value:
+  - 144
+  - 144
+crop_down_offset:
+  desc: null
+  value: 48
+color_crop_type:
+  desc: null
+  value: null
+depth_crop_type:
+  desc: null
+  value: null
+segmask_crop_type:
+  desc: null
+  value: null
+add_crop_binary_mask:
+  desc: null
+  value: false
+add_coord_conv_map:
+  desc: null
+  value: false
+use_context_color:
+  desc: null
+  value: false
+use_context_depth:
+  desc: null
+  value: false
+use_context_segmask:
+  desc: null
+  value: false
+context_color_crop_type:
+  desc: null
+  value: null
+context_depth_crop_type:
+  desc: null
+  value: null
+context_segmask_crop_type:
+  desc: null
+  value: null
+context_add_crop_binary_mask:
+  desc: null
+  value: false
+context_add_coord_conv_map:
+  desc: null
+  value: false
+use_contact_map:
+  desc: null
+  value: false
+use_sdf_maps:
+  desc: null
+  value: false
+use_normals_maps:
+  desc: null
+  value: false
+which_objects:
+  desc: null
+  value: both
+max_contact_prob:
+  desc: null
+  value: 0.1
+max_depth:
+  desc: null
+  value: 2.0
+grasped_dtc_max_value:
+  desc: null
+  value: 0.2
+env_dtc_max_value:
+  desc: null
+  value: 0.4
+grasped_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.2
+env_normals_mask_max_dtc_value:
+  desc: null
+  value: 0.4
+clamp_dtc:
+  desc: null
+  value: true
+dtc_adaptive_normalization:
+  desc: null
+  value: false
+mask_normals_within_sdf:
+  desc: null
+  value: true
+adaptive_normals_mask:
+  desc: null
+  value: true
+learnable_contact_preprocess_params:
+  desc: null
+  value: true
+contact_model_name:
+  desc: null
+  value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path:
+  desc: null
+  value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type:
+  desc: null
+  value: small
+debug_timestamps:
+  desc: null
+  value: false
+open_loop:
+  desc: null
+  value: false
+action_trajectories:
+  desc: null
+  value: true
+stop_after_action:
+  desc: null
+  value: false
+interpolation_frequency:
+  desc: null
+  value: 25
+policy_frequency:
+  desc: null
+  value: 5
+wait_for_new_camera_frames:
+  desc: null
+  value: true
+baseline:
+  desc: null
+  value: false
+train_demo_idxs_list_or_num:
+  desc: null
+  value: -1
+log_train_every_steps:
+  desc: null
+  value: 25
+name_of_expert_demo:
+  desc: null
+  value: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+store_dataset_in_memory:
+  desc: null
+  value: false
+expert_dataset:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+action_key:
+  desc: null
+  value: action_trajectory_25hz
+semantic_demo_grouping_name:
+  desc: null
+  value: semantic_demo_grouping.yaml
+semantic_demo_grouping:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml
+include_groups_list:
+  desc: null
+  value:
+  - modelsys_twodim
+expert_dataset_config:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml
+name_of_valid_demo:
+  desc: null
+  value: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+valid_demo_idxs_list_or_num:
+  desc: null
+  value: null
+val_num_groups:
+  desc: null
+  value: 0
+load_bc:
+  desc: null
+  value: true
+checkpoint_epoch_list:
+  desc: null
+  value:
+  - 99
+  - 199
+  - 299
+  - 399
+  - 499
+  - 599
+  - 699
+  - 799
+  - 899
+  - 999
+  - 1249
+  - 1499
+  - 1749
+  - 1999
+  - 2999
+  - 3999
+  - 4999
+  - 5999
+  - 6999
+  - 7999
+  - 8999
+  - 9999
+snapshot_root_dir:
+  desc: null
+  value: /mnt/kostas-graid/datasets/extrinsic_contact_data/FISH
+save_snapshot:
+  desc: null
+  value: true
+save_last_snapshot:
+  desc: null
+  value: true
+save_snapshot_when_done:
+  desc: null
+  value: true
+top_k_checkpoints:
+  desc: null
+  value: 5
+save_snapshot_link_to_weights_dir:
+  desc: null
+  value: deprecated
+bc_regularize:
+  desc: null
+  value: false
+bc_weight_type:
+  desc: null
+  value: qfilter
+experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/510_0
+agent:
+  desc: null
+  value:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgent
+    name: diffusion_policy
+    load_checkpoint: true
+    device: cuda
+    n_obs_steps: 1
+    suite_name: frankagym
+    obs_type: pixels
+    enable_arm: true
+    enable_camera: true
+    use_tb: true
+    desired_image_shape:
+    - 13
+    - 180
+    - 240
+    orig_cam_shape:
+    - 3
+    - 240
+    - 320
+    config:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: cuda
+      cam_resize_shape:
+      - 13
+      - 180
+      - 240
+      orig_cam_shape:
+      - 3
+      - 240
+      - 320
+      policy_frequency: 5
+      interpolation_frequency: 25
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: 36
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: 5
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: 1000000
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: true
+        use_color: true
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: false
+          representation: channels
+          mask_list:
+          - EE_obj_mask
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: null
+          depth_crop_type: null
+          segmask_crop_type: null
+          crop_hw:
+          - 144
+          - 144
+          crop_down_offset: 48
+          add_crop_binary_mask: false
+          add_coord_conv_map: false
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: false
+          use_depth: false
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: false
+        use_sdf_maps: false
+        use_normals_maps: false
+        which_objects: both
+        grasped_dtc_max_value: 0.2
+        env_dtc_max_value: 0.4
+        grasped_normals_mask_max_dtc_value: 0.2
+        env_normals_mask_max_dtc_value: 0.4
+        clamp_dtc: true
+        max_contact_prob: 0.1
+        mask_normals_within_sdf: true
+        dtc_adaptive_normalization: false
+        adaptive_normals_mask: true
+        max_depth: 2.0
+        image_shape:
+        - 13
+        - 180
+        - 240
+        learnable_contact_preprocess_params: true
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+        zero_centered: false
+suite:
+  desc: null
+  value:
+    suite: frankagym
+    name: frankagym
+    frame_stack: 1
+    action_repeat: 1
+    discount: 0.99
+    hidden_dim: 1024
+    num_train_frames: 2010
+    num_seed_frames: 260
+    num_train_epochs: 5000
+    validate_every_epochs: 100
+    validate_diffusion_on_action_loss_every_epochs: 500
+    train_eval_diffusion_on_action_loss_every_epochs: 500
+    check_topk_every_epochs: 10
+    save_snapshot_every_epochs: 5000
+    eval_every_frames: 2000
+    num_eval_episodes: 5
+    save_snapshot: true
+    wait_for_user_to_start_episode: true
+    task_make_fn:
+      _target_: suite.frankagym.make
+      name: FrankaInsertion-v1
+      height: 240
+      width: 320
+      frame_stack: 1
+      action_repeat: 1
+      seed: 0
+      enable_arm: true
+      enable_gripper: true
+      start_with_gripper_open: true
+      enable_camera: true
+      path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+      contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+      x_limit:
+      - 0.2
+      - 0.7
+      y_limit:
+      - -0.4
+      - 0.4
+      z_limit:
+      - -0.05
+      - 0.55
+      device: cuda
+      interpolation_frequency: 25
+      policy_frequency: 5
+      debug_timestamps: false
+      stop_after_action: false
+      open_loop: false
+      wait_for_new_camera_frames: true
+      action_key: action_trajectory_25hz
+      action_trajectory_horizon: 36
+      action_trajectories: true
+      path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr
+      agent_policy_cfg:
+        _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+        compile: false
+        device: cuda
+        cam_resize_shape:
+        - 13
+        - 180
+        - 240
+        orig_cam_shape:
+        - 3
+        - 240
+        - 320
+        policy_frequency: 5
+        interpolation_frequency: 25
+        policy_cfg:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+          n_obs_steps: 1
+          horizon: 36
+          n_action_steps: 36
+          output_shapes:
+            action:
+            - 7
+          input_normalization_modes:
+            observation.image: mean_std
+            observation.state: min_max
+            observation.action_history: min_max
+          output_normalization_modes:
+            action: min_max
+          vision_backbone: resnet18
+          pretrained_backbone_weights: null
+          transforms:
+          - _target_: torchaug.transforms.RandomAffine
+            degrees:
+            - -5
+            - 5
+            translate:
+            - 0.05
+            - 0.05
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          - _target_: torchaug.transforms.RandomColorJitter
+            brightness: 0.3
+            contrast: 0.4
+            saturation: 0.5
+            hue: 0.08
+            batch_transform: true
+            num_chunks: -1
+            batch_inplace: true
+          use_group_norm: true
+          spatial_softmax_num_keypoints: 32
+          action_history_encoder_config:
+            _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+            in_channels: 7
+            out_channels: 32
+            history_length: 6
+            kernel_size: 5
+            downsample_kernel_size: 3
+            downsample_stride: 2
+            downsample_padding: 1
+          down_dims:
+          - 256
+          - 512
+          - 1024
+          kernel_size: 5
+          n_groups: 8
+          diffusion_step_embed_dim: 128
+          use_film_scale_modulation: true
+          noise_scheduler_type: DDIM
+          beta_schedule: squaredcos_cap_v2
+          beta_start: 0.0001
+          beta_end: 0.02
+          prediction_type: epsilon
+          clip_sample: true
+          clip_sample_range: 1.0
+          num_train_timesteps: 50
+          num_inference_steps: 10
+          do_mask_loss_for_padding: false
+          input_shapes:
+            observation.image:
+            - 13
+            - 180
+            - 240
+            context_observation.image:
+            - 13
+            - 180
+            - 240
+            observation.state:
+            - 8
+            observation.action_history:
+            - 7
+        train_cfg:
+          _target_: utils.TrainConfig
+          lr: 0.0001
+          lr_scheduler: cosine
+          lr_warmup_steps: 500
+          adam_betas:
+          - 0.95
+          - 0.999
+          adam_eps: 1.0e-08
+          adam_weight_decay: 1.0e-06
+          grad_clip_norm: 10
+          offline_steps: 1000000
+          use_amp: true
+        observation_cfg:
+          _target_: agent.encoder.VisualFeatureSet
+          use_depth: true
+          use_color: true
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: false
+            representation: channels
+            mask_list:
+            - EE_obj_mask
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: null
+            depth_crop_type: null
+            segmask_crop_type: null
+            crop_hw:
+            - 144
+            - 144
+            crop_down_offset: 48
+            add_crop_binary_mask: false
+            add_coord_conv_map: false
+          context_input_config:
+            _target_: agent.encoder.ContextInputConfig
+            use_color: false
+            use_depth: false
+            mask_input_dict:
+              _target_: agent.encoder.MaskInputDict
+              enable: false
+              representation: channels
+              mask_list:
+              - EE_obj_mask
+            crop_input_config:
+              _target_: agent.encoder.CropInputConfig
+              color_crop_type: null
+              depth_crop_type: null
+              segmask_crop_type: null
+              crop_hw:
+              - 144
+              - 144
+              crop_down_offset: 48
+              add_crop_binary_mask: false
+              add_coord_conv_map: false
+          mask_soft_approx_scheduler_config:
+            _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+            num_steps: 40000
+            initial_value: 10.0
+            final_value: 1000.0
+            interpolation_scheme: cosine
+          use_contact_map: false
+          use_sdf_maps: false
+          use_normals_maps: false
+          which_objects: both
+          grasped_dtc_max_value: 0.2
+          env_dtc_max_value: 0.4
+          grasped_normals_mask_max_dtc_value: 0.2
+          env_normals_mask_max_dtc_value: 0.4
+          clamp_dtc: true
+          max_contact_prob: 0.1
+          mask_normals_within_sdf: true
+          dtc_adaptive_normalization: false
+          adaptive_normals_mask: true
+          max_depth: 2.0
+          image_shape:
+          - 13
+          - 180
+          - 240
+          learnable_contact_preprocess_params: true
+          learning_rate: 0.0001
+          weight_decay: 0.0
+          contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+          zero_centered: false
+      true_action_history: false
+num_train_frames_bc:
+  desc: null
+  value: 50000
+num_train_frames_drq:
+  desc: null
+  value: 1100000
+stddev_schedule_drq:
+  desc: null
+  value: linear(1.0,0.1,100000)
+task_name:
+  desc: null
+  value: FrankaInsertion-v1
+num_train_frames_vinn:
+  desc: null
+  value: 25000
+num_train_frames_diffusion:
+  desc: null
+  value: 1000000
+num_train_epochs_bc:
+  desc: null
+  value: 5000
+num_train_epochs_diffusion:
+  desc: null
+  value: 15000
+validate_every_epochs_bc:
+  desc: null
+  value: 5
+validate_every_epochs_diffusion:
+  desc: null
+  value: 250
+validate_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+train_eval_diffusion_on_action_loss_every_epochs:
+  desc: null
+  value: 250
+check_topk_every_epochs:
+  desc: null
+  value: 5
+check_topk_every_epochs_diffusion:
+  desc: null
+  value: 250
+save_snapshot_every_epochs_diffusion:
+  desc: null
+  value: 1500
+x_limit:
+  desc: null
+  value:
+  - 0.2
+  - 0.7
+y_limit:
+  desc: null
+  value:
+  - -0.4
+  - 0.4
+z_limit:
+  desc: null
+  value:
+  - -0.05
+  - 0.55
+home_displacement:
+  desc: null
+  value:
+  - 0.55
+  - 0.0
+  - 0.55
+  - 180.0
+  - 0.0
+  - 0.0
+enable_gripper:
+  desc: null
+  value: true
+start_with_gripper_open:
+  desc: null
+  value: true
+offset_mask:
+  desc: null
+  value:
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+  - 1
+path_to_depth_extrinsics:
+  desc: null
+  value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type:
+  desc: null
+  value: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+save_buffer:
+  desc: null
+  value: true
+num_eval:
+  desc: null
+  value: 5
+random_start:
+  desc: null
+  value: false
+eval_starts:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1
+num_valid_demos:
+  desc: null
+  value: null
+load_checkpoint:
+  desc: null
+  value: true
+checkpoint_epoch:
+  desc: null
+  value: 12000
+load_residual_weight:
+  desc: null
+  value: false
+checkpoint_root_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH
+checkpoint_weight_dir:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0
+residual_weight:
+  desc: null
+  value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt
+final_experiment_dir:
+  desc: null
+  value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948
+_wandb:
+  desc: null
+  value:
+    code_path: code/FISH/eval_robot.py
+    python_version: 3.10.14
+    cli_version: 0.17.5
+    framework: torch
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1734491394
+    t:
+      1:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      2:
+      - 1
+      - 41
+      - 49
+      - 50
+      - 55
+      - 83
+      3:
+      - 16
+      - 23
+      - 35
+      4: 3.10.14
+      5: 0.17.5
+      8:
+      - 5
+      13: linux-x86_64
+grasped_obj_name:
+  desc: null
+  value: modelsys
+left_book_slot:
+  desc: null
+  value: twodim

220948/wandb/run-20241217_220954-7lp0oxke/files/diff.patch

@@ -0,0 +1,195 @@
+diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml
+index bb0a413..bca75f8 100644
+--- a/FISH/cfgs/config_eval.yaml
++++ b/FISH/cfgs/config_eval.yaml
+@@ -107,7 +107,7 @@ num_valid_demos: null
+ val_num_groups: 3
+ 
+ # name_of_expert_demo: 'book_insertion_wo_ends_wedge_v2_76_all_demos_10hz_no_filter_wo_flow_mask_expert_demos_imp_act'
+-name_of_expert_demo: '192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act'
++name_of_expert_demo: '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act'
+ expert_dataset_dirpath: '${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}'
+ 
+ expert_dataset: '${expert_dataset_dirpath}/demos.zarr'
+@@ -123,89 +123,28 @@ bc_weight_type: 'qfilter' # linear, qfilter
+ 
+ # Load weights
+ load_checkpoint: ${agent.load_checkpoint}
+-# 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# full 32 demos
+-# wandb_run_id: "265625" # RGBD history seed 0
+-# wandb_run_id: "266370_1" # RGBD history seed 1
+-# wandb_run_id: "266370_2" # RGBD history seed 2
+-# wandb_run_id: "266382_4" # RGBD history seed 4
+-
+-# 16 held out demos (16 demos)
+-# wandb_run_id: "266392_0" # RGBD+act history dataset seed 0
+-# wandb_run_id: "266407_0" # RGBD+act history dataset seed 1
+-wandb_run_id: "266406_0" # RGBD+act history dataset seed 2
+-
+-# wandb_run_id: "265626" # RGBD+Mask+act history seed 0
+-# wandb_run_id: "265638" # RGBD+Mask+act history seed 1
+-# wandb_run_id: "265637" # RGBD+Mask+act history seed 2
+-# wandb_run_id: "265636" # RGBD+Mask+act history seed 3
+-# wandb_run_id: "265635" # RGBD+Mask+act history seed 4
+-
+-# wandb_run_id: "265628" # RGBD+Mask+contact+act history seed 0
+-# wandb_run_id: "265630" # RGBD+Mask+contact+act history seed 1
+-# wandb_run_id: "265631" # RGBD+Mask+contact+act history seed 2
+-# wandb_run_id: "265633" # RGBD+Mask+contact+act history seed 3
+-# wandb_run_id: "265634" # RGBD+Mask+contact+act history seed 4
+-
+-# 8 held out demos (26 demos)
+-# wandb_run_id: "265652" # RGBD+Mask+contact+act history dataset seed 0
+-# wandb_run_id: "265654" # RGBD+Mask+contact+act history dataset seed 1
+-# wandb_run_id: "265655" # RGBD+Mask+contact+act history dataset seed 2
+-
+-# 8 held out demos (26 demos)
+-# wandb_run_id: "265656" # RGBD+Mask+act history dataset seed 0
+-# wandb_run_id: "265657" # RGBD+Mask+act history dataset seed 1
+-# wandb_run_id: "265658" # RGBD+Mask+act history dataset seed 2
+-
+-# 16 held out demos (16 demos)
+-# wandb_run_id: "265641" # RGBD+Mask+act history dataset seed 0
+-# wandb_run_id: "265644" # RGBD+Mask+act history dataset seed 1
+-# wandb_run_id: "265645" # RGBD+Mask+act history dataset seed 2
+-# wandb_run_id: "266381_0" # RGBD+Mask+act history dataset seed 4
+-
+-# 16 held out demos (16 demos)
+-# wandb_run_id: "265646" # RGBD+Mask+Contact+act history dataset seed 0
+-# wandb_run_id: "265649" # RGBD+Mask+Contact+act history dataset seed 1
+-# wandb_run_id: "265650" # RGBD+Mask+Contact+act history dataset seed 2
+-# wandb_run_id: "266378_0" # RGBD+Mask+Contact+act history dataset seed 4
+-
+-
+-
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# full dataset
+-# 6 step action history
+-# wandb_run_id: '250358_0' # RGBD+act history
+-# wandb_run_id: '250371_0' # RGBD+mask+act history
+-# wandb_run_id: '250366_0' # RGBD+contact+act history
+-# wandb_run_id: '250376_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250366_0' # RGBD+contact+act history
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# 16 held out demos
+-# wandb_run_id: '250362_0' # RGBD+act history
+-# wandb_run_id: '250373_0' # RGBD+mask+act history
+-# wandb_run_id: '250378_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250368_0' # RGBD+contact+act history
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# 32 held out demos
+-# wandb_run_id: '250365_0' # RGBD+act history
+-# wandb_run_id: '250375_0' # RGBD+mask+act history
+-wandb_run_id: '250380_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250370_0' # RGBD+contact+act history
+-# 4 step action history
+-# wandb_run_id: '264772_0' # RGBD+act history
+-# wandb_run_id: '264775_0' # RGBD+mask+act history
+-# wandb_run_id: '264776_0' # RGBD+contact+act history
+-# wandb_run_id: '264777_0' # RGBD+mask+contact+act history
+-
+-# 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+-# 16 held out demos
+-# wandb_run_id: '250362_0' # RGBD+act history
+-# wandb_run_id: '250373_0' # RGBD+mask+act history
+-# wandb_run_id: '250378_0' # RGBD+mask+contact+act history
+-# wandb_run_id: '250368_0' # RGBD+contact+act history
++# 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
++# 14 demos
++
++# RGBD+act history 
++# wandb_run_id: '497_0' # german
++# wandb_run_id: '505_0' # hbm
++# wandb_run_id: '506_0' # lady
++# wandb_run_id: '507_0' # greece
++# wandb_run_id: '508_0' # fowlers
++# wandb_run_id: '509_0' # lib
++wandb_run_id: '510_0' # modelsys
++# wandb_run_id: '511_0' # electrodyn
++
++# RGBD+mask+act history
++# wandb_run_id: '512_0' # german
++# wandb_run_id: '520_0' # hbm
++# wandb_run_id: '521_0' # lady
++# wandb_run_id: '522_0' # greece
++# wandb_run_id: '523_0' # fowlers
++# wandb_run_id: '524_0' # lib
++# wandb_run_id: '525_0' # modelsys
++# wandb_run_id: '526_0' # electrodyn
+ 
+ # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+ # 32 held out demos
+diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py
+index 9406a46..0f225ec 100644
+--- a/FISH/download_model_checkpoints.py
++++ b/FISH/download_model_checkpoints.py
+@@ -15,12 +15,24 @@ if not local_checkpoint_root_dir.exists():
+     local_checkpoint_root_dir.mkdir(parents=True)
+ 
+ # Download the model checkpoints
+-remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1')
++# remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1')
++remote_checkpoint_root_dir = Path('/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1')
+ remote_checkpoint_root_dir = remote_checkpoint_root_dir.expanduser()
+ remote_username = 'leonmkim'
+-remote_host = 'halley'
++remote_host = 'grasp-login1'
+ 
+ #180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
++# run_id_list = [
++#     '512_0', # german
++#     '520_0', # hbm
++#     '521_0', # lady
++#     '522_0', # greece
++#     '523_0', # fowlers
++#     '524_0', # lib
++#     '525_0', # modelsys
++#     '526_0', # electrodyn
++# ]
++
+ # run_id_list = [
+ #     '265638',
+ #     '265637',
+@@ -75,6 +87,17 @@ remote_host = 'halley'
+ # ]
+ 
+ # 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
++# run_id_list = [
++#     '497_0', # german
++#     '505_0', # hbm
++#     '506_0', # lady
++#     '507_0', # greece
++#     '508_0', # fowlers
++#     '509_0', # lib
++#     '510_0', # modelsys
++#     '511_0', # electrodyn
++# ]
++
+ # run_id_list = [
+ #     '266382_4',
+ #     '266370_2',
+@@ -116,13 +139,13 @@ remote_host = 'halley'
+ # ]
+ 
+ # 180x240_1_RGB_D_2.0_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+-run_id_list = [
+-    # '266384_4,# not yet available
+-    '266372_1',
+-    '266372_2',
+-    '265627',
+-    '265620',
+-]
++# run_id_list = [
++#     # '266384_4,# not yet available
++#     '266372_1',
++#     '266372_2',
++#     '265627',
++#     '265620',
++# ]
+ checkpoint_epoch = 12000
+ 
+ # use subprocess to download the model checkpoints in parallel

220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/0_eval_0_f8fef5727eff9ff71143.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8fef5727eff9ff711434a6872b2606d4abe372028bd219ceb58dfa95c013ee2
+size 1152439

220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/1_eval_1_8a67326b5dfefa361130.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a67326b5dfefa3611301ecb0f2a2100e831e8d33dfa72fcc6d707320e8bfc17
+size 1160665

220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/2_eval_2_4735c6188bf9f36869cb.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4735c6188bf9f36869cb28f87bdc9cb4676365377b2b1bd34c01893841b876db
+size 1227110

220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/3_eval_3_a4af24fd7d85bfc8c826.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4af24fd7d85bfc8c826ed9fb0677ef3b356f3a74873bf3e4f95489ce5d37e74
+size 1237139

220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/4_eval_4_ccefe4eb2a70fd4856d6.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccefe4eb2a70fd4856d6e54f10fa162d2eb75a982b20f5d9e7ca1f796e073ab1
+size 1181573

220948/wandb/run-20241217_220954-7lp0oxke/files/output.log

@@ -0,0 +1,237 @@
+
+loaded agent with feature_type: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1
+[INFO] [1734491400.475237]: resetting environment
+[INFO] [1734491400.477058]: cleared current plan
+[INFO] [1734491400.478149]: moving to home
+[INFO] [1734491401.480470]: reached home
+[INFO] [1734491401.482444]: reset action history
+[INFO] [1734491404.190222]: environment reset
+Starting episode 0
+[INFO] [1734491404.192429]: resetting environment
+[INFO] [1734491404.193192]: cleared current plan
+[INFO] [1734491404.194030]: moving to home
+[INFO] [1734491405.199469]: reached home
+[INFO] [1734491405.200946]: reset action history
+[INFO] [1734491407.908553]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode![INFO] [1734491410.906342]: resetting environment
+[INFO] [1734491410.907622]: cleared current plan
+[INFO] [1734491410.909274]: moving to home
+[INFO] [1734491411.911422]: reached home
+[INFO] [1734491411.912869]: reset action history
+[INFO] [1734491414.622161]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/episode_rosbags/episode_0_2024-12-17-22-10-14.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-82633.log
+started roslaunch server http://158.130.50.37:37269/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-1]: started with pid [82910]
+process[print_text-2]: started with pid [82911]
+process[rosbag_record-3]: started with pid [82934]
+started bagging!
+For topic gripper_width: timestamp difference is -148167862 for nearest: 1734491421015432821 - target: 1734491421163600683 at idx 92
+For topic gripper_width: timestamp difference is -146120254 for nearest: 1734491422015410718 - target: 1734491422161530972 at idx 108
+For topic gripper_width: timestamp difference is -145314615 for nearest: 1734491422415370304 - target: 1734491422560684919 at idx 110
+For topic gripper_width: timestamp difference is -131351919 for nearest: 1734491429215410738 - target: 1734491429346762657 at idx 209
+For topic gripper_width: timestamp difference is -130984019 for nearest: 1734491429415360023 - target: 1734491429546344042 at idx 210
+For topic gripper_width: timestamp difference is -130540691 for nearest: 1734491429615411200 - target: 1734491429745951891 at idx 211
+For topic gripper_width: timestamp difference is -132345389 for nearest: 1734491431015438844 - target: 1734491431147784233 at idx 233
+For topic gripper_width: timestamp difference is -127696107 for nearest: 1734491439615392854 - target: 1734491439743088961 at idx 249
+[INFO] [1734491441.413541]: Storing episode...
+[print_text-2] killing on exit
+[rosbag_record-3] killing on exit[pub_text-1] killing on exit
+[INFO] [1734491442.141660]: Stored episode 1.
+[INFO] [1734491442.146656]: Saving video...
+[INFO] [1734491442.334895]: Video saved!
+Starting episode 1
+[INFO] [1734491442.346773]: resetting environment
+[INFO] [1734491442.347154]: cleared current plan
+[INFO] [1734491442.347660]: moving to home
+[INFO] [1734491445.749259]: reached home
+[INFO] [1734491445.749741]: reset action history
+[INFO] [1734491448.457689]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!
+[INFO] [1734491451.929708]: resetting environment
+[INFO] [1734491451.929891]: cleared current plan
+[INFO] [1734491451.930378]: moving to home
+[INFO] [1734491452.932537]: reached home
+[INFO] [1734491452.933117]: reset action history
+[INFO] [1734491455.640897]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/episode_rosbags/episode_1_2024-12-17-22-10-55.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-82633.log
+started roslaunch server http://158.130.50.37:44533/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-4]: started with pid [83038]
+process[print_text-5]: started with pid [83039]
+process[rosbag_record-6]: started with pid [83040]
+started bagging!
+For topic gripper_width: timestamp difference is -102275498 for nearest: 1734491472615369432 - target: 1734491472717644930 at idx 216
+For topic gripper_width: timestamp difference is -112551358 for nearest: 1734491473615407798 - target: 1734491473727959156 at idx 227
+For topic gripper_width: timestamp difference is -108801612 for nearest: 1734491475415460339 - target: 1734491475524261951 at idx 249
+For topic gripper_width: timestamp difference is -108396383 for nearest: 1734491475615450529 - target: 1734491475723846912 at idx 249
+For topic gripper_width: timestamp difference is -107187762 for nearest: 1734491476215445504 - target: 1734491476322633266 at idx 249
+For topic gripper_width: timestamp difference is -105539144 for nearest: 1734491477015439450 - target: 1734491477120978594 at idx 249
+For topic gripper_width: timestamp difference is -105144285 for nearest: 1734491477215409733 - target: 1734491477320554018 at idx 249
+For topic gripper_width: timestamp difference is -103527499 for nearest: 1734491478015410470 - target: 1734491478118937969 at idx 249
+For topic gripper_width: timestamp difference is -103091465 for nearest: 1734491478215434565 - target: 1734491478318526030 at idx 249
+For topic gripper_width: timestamp difference is -102692113 for nearest: 1734491478415410771 - target: 1734491478518102884 at idx 249
+For topic gripper_width: timestamp difference is -100630680 for nearest: 1734491479415413937 - target: 1734491479516044617 at idx 249
+For topic gripper_width: timestamp difference is -163676167 for nearest: 1734491481148704624 - target: 1734491481312380791 at idx 249
+[INFO] [1734491482.369867]: Storing episode...
+[rosbag_record-6] killing on exit
+[print_text-5] killing on exit
+[pub_text-4] killing on exit
+[INFO] [1734491483.150241]: Stored episode 2.
+[INFO] [1734491483.152280]: Saving video...
+[INFO] [1734491483.711456]: Video saved!
+Starting episode 2
+[INFO] [1734491483.752465]: resetting environment
+[INFO] [1734491483.754940]: cleared current plan
+[INFO] [1734491483.757016]: moving to home
+[WARN] [1734491483.741836]: Plan exhausted
+[INFO] [1734491487.162941]: reached home
+[INFO] [1734491487.164181]: reset action history
+[INFO] [1734491489.893756]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode![INFO] [1734491495.339069]: resetting environment
+[INFO] [1734491495.342220]: cleared current plan
+[INFO] [1734491495.344300]: moving to home
+[INFO] [1734491496.349031]: reached home
+[INFO] [1734491496.350281]: reset action history
+[INFO] [1734491499.115384]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/episode_rosbags/episode_2_2024-12-17-22-11-39.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-82633.log
+started roslaunch server http://158.130.50.37:35143/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-7]: started with pid [83191]
+process[print_text-8]: started with pid [83192]
+process[rosbag_record-9]: started with pid [83193]
+started bagging!
+For topic gripper_width: timestamp difference is -108196361 for nearest: 1734491505115410464 - target: 1734491505223606825 at idx 82
+For topic gripper_width: timestamp difference is -106569833 for nearest: 1734491507115433865 - target: 1734491507222003698 at idx 105
+For topic gripper_width: timestamp difference is -103437951 for nearest: 1734491511115407655 - target: 1734491511218845606 at idx 159
+For topic gripper_width: timestamp difference is -101303537 for nearest: 1734491513715432446 - target: 1734491513816735983 at idx 198
+[INFO] [1734491527.906707]: Storing episode...
+[rosbag_record-9] killing on exit
+[print_text-8] killing on exit
+[pub_text-7] killing on exit
+[INFO] [1734491528.666342]: Stored episode 3.
+[INFO] [1734491528.670540]: Saving video...
+[INFO] [1734491528.871095]: Video saved!
+Starting episode 3
+[INFO] [1734491528.921076]: resetting environment
+[INFO] [1734491528.926922]: cleared current plan
+[INFO] [1734491528.932398]: moving to home
+[INFO] [1734491532.552181]: reached home
+[INFO] [1734491532.557734]: reset action history
+[INFO] [1734491535.338773]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode![INFO] [1734491536.600562]: resetting environment
+[INFO] [1734491536.608042]: cleared current plan
+[INFO] [1734491536.613784]: moving to home
+[INFO] [1734491537.616423]: reached home
+[INFO] [1734491537.620962]: reset action history
+[INFO] [1734491540.397352]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/episode_rosbags/episode_3_2024-12-17-22-12-20.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-82633.log
+started roslaunch server http://158.130.50.37:33721/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-10]: started with pid [83320]
+process[print_text-11]: started with pid [83321]
+process[rosbag_record-12]: started with pid [83322]
+started bagging!
+[INFO] [1734491569.278789]: Storing episode...
+[rosbag_record-12] killing on exit
+[print_text-11] killing on exit
+[pub_text-10] killing on exit
+[INFO] [1734491570.128282]: Stored episode 4.
+[INFO] [1734491570.130187]: Saving video...
+[INFO] [1734491570.348176]: Video saved!
+Starting episode 4
+[INFO] [1734491570.396847]: resetting environment
+[INFO] [1734491570.400203]: cleared current plan
+[INFO] [1734491570.406079]: moving to home
+[INFO] [1734491574.015222]: reached home
+[INFO] [1734491574.023452]: reset action history
+[INFO] [1734491576.823745]: environment reset
+Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode![INFO] [1734491588.856218]: resetting environment
+[INFO] [1734491588.865689]: cleared current plan
+[INFO] [1734491588.868185]: moving to home
+[INFO] [1734491589.876432]: reached home
+[INFO] [1734491589.881403]: reset action history
+[INFO] [1734491592.666529]: environment reset
+bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/episode_rosbags/episode_4_2024-12-17-22-13-12.bag
+... logging to /home/leonmkim/.ros/log/94e09230-bcdd-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-82633.log
+started roslaunch server http://158.130.50.37:39839/
+SUMMARY
+========
+PARAMETERS
+ * /rosdistro: noetic
+ * /rosversion: 1.16.0
+NODES
+  /
+    print_text (rostopic/rostopic)
+    pub_text (rostopic/rostopic)
+    rosbag_record (rosbag/record)
+ROS_MASTER_URI=http://localhost:11311
+process[pub_text-13]: started with pid [83519]
+process[print_text-14]: started with pid [83520]
+process[rosbag_record-15]: started with pid [83521]
+started bagging!
+For topic gripper_width: timestamp difference is -195778477 for nearest: 1734491603115437639 - target: 1734491603311216116 at idx 153
+[INFO] [1734491620.191433]: Storing episode...
+[pub_text-13] killing on exit
+[rosbag_record-15] killing on exit
+[print_text-14] killing on exit
+[INFO] [1734491620.955955]: Stored episode 5.
+[INFO] [1734491620.967240]: Saving video...
+[INFO] [1734491621.184178]: Video saved!
+[INFO] [1734491621.241153]: resetting environment
+[INFO] [1734491621.248760]: cleared current plan
+[INFO] [1734491621.251046]: moving to home
+[INFO] [1734491624.961219]: reached home
+[INFO] [1734491624.966520]: reset action history
+[INFO] [1734491627.825994]: environment reset
+Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success
+proceeding to start episode!

220948/wandb/run-20241217_220954-7lp0oxke/files/requirements.txt

@@ -0,0 +1,339 @@
+Cython==3.0.10
+Farama-Notifications==0.0.4
+GitPython==3.1.43
+Jinja2==3.1.4
+Markdown==3.6
+MarkupSafe==2.1.5
+POT==0.7.0
+PyOpenGL==3.1.7
+PySocks==1.7.1
+PyYAML==6.0.1
+Pygments==2.18.0
+Rtree==1.3.0
+Werkzeug==3.0.3
+absl-py==2.1.0
+accelerate==0.33.0
+actionlib-msgs==1.13.0.post3
+actionlib==1.12.0
+actionlib==1.14.0
+aiohttp==3.9.5
+aiosignal==1.3.1
+angles==1.9.13
+antlr4-python3-runtime==4.9.3
+anyio==4.4.0
+asciitree==0.3.3
+async-timeout==4.0.3
+attrs==23.2.0
+autoprop==4.1.0
+beartype==0.18.5
+beautifulsoup4==4.12.3
+bondpy==1.8.6
+byol-pytorch==0.8.0
+cachetools==5.4.0
+camera-calibration-parsers==1.12.0
+camera-calibration==1.17.0
+cascadio==0.0.13
+catkin-pkg==1.0.0
+catkin==0.7.18
+catkin==0.8.10
+certifi==2024.7.4
+cffi==1.16.0
+chardet==5.2.0
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+cmake==3.30.1
+colorlog==6.8.2
+contourpy==1.2.1
+controller-manager-msgs==0.20.0
+controller-manager==0.20.0
+coverage==7.6.0
+coveralls==4.0.1
+cv-bridge==1.16.2
+cycler==0.12.1
+datasets==2.20.0
+decorator==4.4.2
+deepdiff==7.0.1
+defusedxml==0.7.1
+diagnostic-analysis==1.11.0
+diagnostic-common-diagnostics==1.11.0
+diagnostic-updater==1.11.0
+diffusers==0.27.2
+dill==0.3.8
+distro==1.9.0
+dm-control==1.0.8
+dm-env==1.6
+dm-tree==0.1.8
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.21.2
+dynamic-reconfigure==1.7.3
+einops==0.8.0
+embreex==2.17.7.post5
+empy==3.3.4
+etils==1.7.0
+exceptiongroup==1.2.2
+ezdxf==1.3.2
+fasteners==0.19
+filelock==3.15.4
+fonttools==4.53.1
+freetype-py==2.4.0
+frozenlist==1.4.1
+fsspec==2024.5.0
+gazebo_plugins==2.9.2
+gazebo_ros==2.9.2
+gdown==5.2.0
+gencpp==0.7.0
+geneus==3.0.0
+genlisp==0.4.18
+genmsg==0.5.12
+genmsg==0.6.0
+gennodejs==2.0.2
+genpy==0.6.14
+genpy==0.6.15
+geometry-msgs==1.13.0.post2
+gitdb==4.0.11
+glfw==2.7.0
+glooey==0.3.6
+gmsh==4.12.2
+gnupg==2.3.1
+google-auth-oauthlib==1.0.0
+google-auth==2.32.0
+grpcio==1.65.1
+gym-envs==0.0.1
+gym-notices==0.0.8
+gym==0.22.0
+gymnasium==0.29.1
+h11==0.14.0
+h5py==3.11.0
+hf_transfer==0.1.8
+httpcore==1.0.5
+httpx==0.27.0
+huggingface-hub==0.23.5
+hydra-core==1.3.2
+hydra-submitit-launcher==1.2.0
+idna==3.7
+image-geometry==1.16.2
+imageio-ffmpeg==0.5.1
+imageio==2.34.2
+importlib_metadata==8.2.0
+importlib_resources==6.4.0
+iniconfig==2.0.0
+interactive-markers==1.12.0
+joint-state-publisher-gui==1.15.1
+joint-state-publisher==1.15.1
+jsonschema-specifications==2023.12.1
+jsonschema==4.23.0
+kiwisolver==1.4.5
+kornia==0.7.3
+kornia_rs==0.1.5
+labmaze==1.0.6
+laser_geometry==1.6.7
+lazy_loader==0.4
+lerobot==0.1.0
+lightning-utilities==0.11.6
+llvmlite==0.43.0
+lxml==5.2.2
+manifold3d==2.5.1
+mapbox-earcut==1.0.1
+markdown-it-py==3.0.0
+matplotlib==3.9.1
+mdurl==0.1.2
+meshio==5.3.5
+message-filters==1.16.0
+more-itertools==10.3.0
+moviepy==1.0.3
+mpmath==1.3.0
+mujoco==3.2.0
+multidict==6.0.5
+multiprocess==0.70.16
+natsort==8.4.0
+netifaces==0.11.0
+networkx==3.3
+nodeenv==1.9.1
+numba==0.60.0
+numcodecs==0.13.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+oauthlib==3.2.2
+omegaconf==2.3.0
+openctm==0.0.5
+opencv-python==4.10.0.84
+ordered-set==4.1.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+pip==24.2
+platformdirs==4.2.2
+pluggy==1.5.0
+proglog==0.1.10
+protobuf==5.27.2
+psutil==6.0.0
+pyarrow-hotfix==0.6
+pyarrow==17.0.0
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pyav==12.3.0
+pycollada==0.8
+pycparser==2.22
+pycryptodomex==3.21.0
+pyglet==1.5.29
+pyinstrument==4.6.2
+pymunk==6.8.1
+pyparsing==2.4.7
+pyrealsense2==2.54.2.5684
+pyribbit==0.1.46
+pyright==1.1.373
+pytest-beartype==0.0.2
+pytest-cov==5.0.0
+pytest==8.3.1
+python-dateutil==2.9.0.post0
+python-fcl==0.7.0.6
+python-qt-binding==0.4.4
+pytorch-lightning==2.4.0
+pytz==2024.1
+qt-dotgraph==0.4.2
+qt-gui-cpp==0.4.2
+qt-gui-py-common==0.4.2
+qt-gui==0.4.2
+referencing==0.35.1
+regex==2024.5.15
+requests-oauthlib==2.0.0
+requests==2.32.3
+rerun-sdk==0.17.0
+resource_retriever==1.12.7
+rich==13.7.1
+ros-numpy==0.0.5
+rosbag==1.16.0
+rosboost-cfg==1.15.8
+rosclean==1.15.8
+roscpp==1.15.11
+roscreate==1.15.8
+rosgraph-msgs==1.11.3.post2
+rosgraph==1.15.11
+rosgraph==1.16.0
+roslaunch==1.16.0
+roslib==1.14.7.post0
+roslib==1.15.8
+roslint==0.12.0
+roslz4==1.16.0
+rosmake==1.15.8
+rosmaster==1.16.0
+rosmsg==1.16.0
+rosnode==1.16.0
+rosparam==1.16.0
+rospkg==1.5.1
+rospy==1.15.11
+rospy==1.16.0
+rosservice==1.16.0
+rostest==1.16.0
+rostopic==1.16.0
+rosunit==1.15.8
+roswtf==1.16.0
+rpds-py==0.19.1
+rqt-console==0.4.12
+rqt-image-view==0.4.17
+rqt-logger-level==0.4.12
+rqt-moveit==0.5.11
+rqt-reconfigure==0.5.5
+rqt-robot-dashboard==0.5.8
+rqt-robot-monitor==0.5.15
+rqt-runtime-monitor==0.5.10
+rqt-rviz==0.7.0
+rqt-tf-tree==0.6.4
+rqt_action==0.4.9
+rqt_bag==0.5.1
+rqt_bag_plugins==0.5.1
+rqt_dep==0.4.12
+rqt_graph==0.4.14
+rqt_gui==0.5.3
+rqt_gui_py==0.5.3
+rqt_launch==0.4.9
+rqt_msg==0.4.10
+rqt_nav_view==0.5.7
+rqt_plot==0.4.13
+rqt_pose_view==0.5.11
+rqt_publisher==0.4.10
+rqt_py_common==0.5.3
+rqt_py_console==0.4.10
+rqt_robot_steering==0.5.12
+rqt_service_caller==0.4.10
+rqt_shell==0.4.11
+rqt_srv==0.4.9
+rqt_top==0.4.10
+rqt_topic==0.4.13
+rqt_web==0.4.10
+rsa==4.9
+ruff==0.5.4
+rviz==1.14.25
+safetensors==0.4.3
+scikit-image==0.24.0
+scikit-video==1.1.11
+scipy==1.14.0
+seaborn==0.13.2
+sensor-msgs==1.13.1
+sentry-sdk==2.11.0
+setproctitle==1.3.3
+setuptools==65.5.0
+shapely==2.0.5
+signature_dispatch==1.0.1
+six==1.16.0
+smach-ros==2.5.2
+smach==2.5.2
+smclib==1.8.6
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+std-msgs==0.5.13.post0
+submitit==1.5.1
+svg.path==6.3
+sympy==1.13.1
+tensorboard-data-server==0.7.2
+tensorboard==2.14.0
+termcolor==2.4.0
+tf-conversions==1.13.2
+tf2-geometry-msgs==0.7.7
+tf2-kdl==0.7.7
+tf2-msgs==0.7.2.post3
+tf2-py==0.7.7
+tf2-ros==0.6.5
+tf2-ros==0.7.7
+tf2_py==0.6.5.post1
+tf==1.13.2
+tifffile==2024.7.24
+tomli==2.0.1
+topic-tools==1.16.0
+torch==2.4.0
+torchaug==0.5.2
+torchmetrics==1.4.0.post0
+torchvision==0.19.0
+tqdm==4.66.4
+trimesh==4.4.3
+triton==3.0.0
+typeguard==3.0.2
+typing_extensions==4.12.2
+tzdata==2024.1
+urchin==0.0.27
+urllib3==2.2.2
+vecrec==0.3.1
+vhacdx==0.0.8.post1
+wandb==0.17.5
+wheel==0.43.0
+xacro==1.14.18
+xatlas==0.0.9
+xxhash==3.4.1
+yarl==1.9.4
+zarr==2.18.2
+zipp==3.19.2
+zstandard==0.23.0

220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-metadata.json

@@ -0,0 +1,92 @@
+{
+    "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31",
+    "python": "3.10.14",
+    "heartbeatAt": "2024-12-18T03:09:54.739757",
+    "startedAt": "2024-12-18T03:09:54.386675",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "agent=diffusion",
+        "suite=frankagym",
+        "suite/frankagym_task@_global_=insertion"
+    ],
+    "state": "running",
+    "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py",
+    "codePathLocal": null,
+    "codePath": "FISH/eval_robot.py",
+    "git": {
+        "remote": "https://github.com/leonmkim/fish_leon.git",
+        "commit": "5f28b54373d7d41e4ebace380ff1080a8ffc4051"
+    },
+    "email": "leonmkim@seas.upenn.edu",
+    "root": "/home/leonmkim/fish_leon",
+    "host": "leonmkim-ROG-Strix-G15CS-G15CS",
+    "username": "leonmkim",
+    "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 3563.9849999999997,
+        "min": 800.0,
+        "max": 4700.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4511.879,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4499.997,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 4500.003,
+            "min": 800.0,
+            "max": 4700.0
+        },
+        {
+            "current": 3000.0,
+            "min": 800.0,
+            "max": 4700.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 915.3232879638672,
+            "used": 410.53357315063477
+        }
+    },
+    "gpu": "NVIDIA GeForce RTX 2070 SUPER",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA GeForce RTX 2070 SUPER",
+            "memory_total": 8589934592
+        }
+    ],
+    "memory": {
+        "total": 62.71595764160156
+    }
+}

220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json

@@ -0,0 +1 @@
+{"eval/0_eval": {"_type": "video-file", "sha256": "f8fef5727eff9ff711434a6872b2606d4abe372028bd219ceb58dfa95c013ee2", "size": 1152439, "path": "media/videos/eval/0_eval_0_f8fef5727eff9ff71143.mp4"}, "global_step": 660, "_timestamp": 1734491629.9142718, "_runtime": 235.51990580558777, "_step": 9, "eval/1_eval": {"_type": "video-file", "sha256": "8a67326b5dfefa3611301ecb0f2a2100e831e8d33dfa72fcc6d707320e8bfc17", "size": 1160665, "path": "media/videos/eval/1_eval_1_8a67326b5dfefa361130.mp4"}, "eval/num_success": 5.0, "episode": 5.0, "eval/success_rate": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "4735c6188bf9f36869cb28f87bdc9cb4676365377b2b1bd34c01893841b876db", "size": 1227110, "path": "media/videos/eval/2_eval_2_4735c6188bf9f36869cb.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "a4af24fd7d85bfc8c826ed9fb0677ef3b356f3a74873bf3e4f95489ce5d37e74", "size": 1237139, "path": "media/videos/eval/3_eval_3_a4af24fd7d85bfc8c826.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "ccefe4eb2a70fd4856d6e54f10fa162d2eb75a982b20f5d9e7ca1f796e073ab1", "size": 1181573, "path": "media/videos/eval/4_eval_4_ccefe4eb2a70fd4856d6.mp4"}, "_wandb": {"runtime": 235}}

220948/wandb/run-20241217_220954-7lp0oxke/logs/debug-internal.log

@@ -0,0 +1,680 @@
+2024-12-17 22:09:54,394 INFO    StreamThr :82813 [internal.py:wandb_internal():85] W&B internal server running at pid: 82813, started at: 2024-12-17 22:09:54.393760
+2024-12-17 22:09:54,395 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status
+2024-12-17 22:09:54,410 INFO    WriterThread:82813 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/run-7lp0oxke.wandb
+2024-12-17 22:09:54,410 DEBUG   SenderThread:82813 [sender.py:send():379] send: header
+2024-12-17 22:09:54,418 DEBUG   SenderThread:82813 [sender.py:send():379] send: run
+2024-12-17 22:09:54,634 INFO    SenderThread:82813 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files
+2024-12-17 22:09:54,634 INFO    SenderThread:82813 [sender.py:_start_run_threads():1188] run started: 7lp0oxke with start time 1734491394.394366
+2024-12-17 22:09:54,650 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: check_version
+2024-12-17 22:09:54,650 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: check_version
+2024-12-17 22:09:54,721 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: run_start
+2024-12-17 22:09:54,725 DEBUG   HandlerThread:82813 [system_info.py:__init__():26] System info init
+2024-12-17 22:09:54,726 DEBUG   HandlerThread:82813 [system_info.py:__init__():41] System info init done
+2024-12-17 22:09:54,726 INFO    HandlerThread:82813 [system_monitor.py:start():194] Starting system monitor
+2024-12-17 22:09:54,726 INFO    SystemMonitor:82813 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-12-17 22:09:54,726 INFO    HandlerThread:82813 [system_monitor.py:probe():214] Collecting system info
+2024-12-17 22:09:54,727 INFO    SystemMonitor:82813 [interfaces.py:start():188] Started cpu monitoring
+2024-12-17 22:09:54,727 INFO    SystemMonitor:82813 [interfaces.py:start():188] Started disk monitoring
+2024-12-17 22:09:54,727 INFO    SystemMonitor:82813 [interfaces.py:start():188] Started gpu monitoring
+2024-12-17 22:09:54,728 INFO    SystemMonitor:82813 [interfaces.py:start():188] Started memory monitoring
+2024-12-17 22:09:54,728 INFO    SystemMonitor:82813 [interfaces.py:start():188] Started network monitoring
+2024-12-17 22:09:54,739 DEBUG   HandlerThread:82813 [system_info.py:probe():152] Probing system
+2024-12-17 22:09:54,741 DEBUG   HandlerThread:82813 [system_info.py:_probe_git():137] Probing git
+2024-12-17 22:09:54,745 DEBUG   HandlerThread:82813 [system_info.py:_probe_git():145] Probing git done
+2024-12-17 22:09:54,745 DEBUG   HandlerThread:82813 [system_info.py:probe():200] Probing system done
+2024-12-17 22:09:54,745 DEBUG   HandlerThread:82813 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-18T03:09:54.739757', 'startedAt': '2024-12-18T03:09:54.386675', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '5f28b54373d7d41e4ebace380ff1080a8ffc4051'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3563.9849999999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4511.879, 'min': 800.0, 'max': 4700.0}, {'current': 4499.997, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.003, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 410.53357315063477}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}}
+2024-12-17 22:09:54,746 INFO    HandlerThread:82813 [system_monitor.py:probe():224] Finished collecting system info
+2024-12-17 22:09:54,746 INFO    HandlerThread:82813 [system_monitor.py:probe():227] Publishing system info
+2024-12-17 22:09:54,746 DEBUG   HandlerThread:82813 [system_info.py:_save_code():44] Saving code
+2024-12-17 22:09:54,747 DEBUG   HandlerThread:82813 [system_info.py:_save_code():67] Saving code done
+2024-12-17 22:09:54,747 DEBUG   HandlerThread:82813 [system_info.py:_save_patches():84] Saving git patches
+2024-12-17 22:09:54,849 DEBUG   HandlerThread:82813 [system_info.py:_save_patches():126] Saving git patches done
+2024-12-17 22:09:54,850 INFO    HandlerThread:82813 [system_monitor.py:probe():229] Finished publishing system info
+2024-12-17 22:09:54,853 DEBUG   SenderThread:82813 [sender.py:send():379] send: files
+2024-12-17 22:09:54,853 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-12-17 22:09:54,853 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now
+2024-12-17 22:09:54,854 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file diff.patch with policy now
+2024-12-17 22:09:55,082 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: python_packages
+2024-12-17 22:09:55,082 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: python_packages
+2024-12-17 22:09:55,085 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:09:55,085 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:09:55,087 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:09:55,087 INFO    HandlerThread:82813 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord {
+  log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/tb"
+  save: true
+}
+control {
+  relay_id: "fda16768da264ac7b1b49c3533ff68be"
+}
+_info {
+  stream_id: "7lp0oxke"
+}
+
+2024-12-17 22:09:55,087 DEBUG   HandlerThread:82813 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml
+2024-12-17 22:09:55,106 INFO    wandb-upload_0:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/0gzqtvqw-wandb-metadata.json
+2024-12-17 22:09:55,119 INFO    wandb-upload_2:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/0iyvg9r1-diff.patch
+2024-12-17 22:09:55,156 INFO    wandb-upload_1:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/3inf5emz-code/FISH/eval_robot.py
+2024-12-17 22:09:55,215 DEBUG   SenderThread:82813 [sender.py:send():379] send: telemetry
+2024-12-17 22:09:55,215 DEBUG   SenderThread:82813 [sender.py:send():379] send: tbrecord
+2024-12-17 22:09:55,216 DEBUG   SenderThread:82813 [sender.py:send():379] send: files
+2024-12-17 22:09:55,216 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0 with policy live
+2024-12-17 22:09:55,413 INFO    wandb-upload_0:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/aljati01-events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0
+2024-12-17 22:09:55,636 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/requirements.txt
+2024-12-17 22:09:55,636 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-metadata.json
+2024-12-17 22:09:55,636 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/diff.patch
+2024-12-17 22:09:55,636 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0
+2024-12-17 22:09:55,636 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:09:55,636 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/code/FISH/eval_robot.py
+2024-12-17 22:09:55,636 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/code/FISH
+2024-12-17 22:09:55,636 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/code
+2024-12-17 22:09:56,082 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:09:57,083 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:09:57,635 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:09:58,083 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:09:59,083 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:00,082 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:10:00,083 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:00,475 DEBUG   SenderThread:82813 [sender.py:send():379] send: config
+2024-12-17 22:10:01,083 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:01,636 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:02,084 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:03,084 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:03,637 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:04,084 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:05,083 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:10:05,084 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:05,637 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:06,084 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:07,084 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:07,638 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:08,084 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:09,084 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:09,639 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:10,082 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:10:10,082 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:10:10,125 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:10,229 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:10:11,085 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:12,085 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:13,085 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:13,640 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:14,085 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:15,086 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:15,640 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:16,084 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:10:16,086 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:17,086 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:17,641 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:18,087 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:19,086 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:20,086 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:21,084 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:10:21,086 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:22,087 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:23,087 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:23,643 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:24,087 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:25,082 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:10:25,082 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:10:25,129 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:26,087 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:26,183 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:10:26,644 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/config.yaml
+2024-12-17 22:10:27,087 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:28,087 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:29,087 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:30,087 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:31,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:31,645 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:32,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:32,253 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:10:33,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:33,646 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:34,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:35,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:36,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:37,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:38,084 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:10:38,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:39,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:40,082 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:10:40,083 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:10:40,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:41,093 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:41,648 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:42,090 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:42,345 DEBUG   SenderThread:82813 [sender.py:send():379] send: files
+2024-12-17 22:10:42,345 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_f8fef5727eff9ff71143.mp4 with policy now
+2024-12-17 22:10:42,345 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 22:10:42,346 DEBUG   SenderThread:82813 [sender.py:send():379] send: history
+2024-12-17 22:10:42,346 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:10:42,347 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:10:42,597 INFO    wandb-upload_1:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/iz3o4t06-media/videos/eval/0_eval_0_f8fef5727eff9ff71143.mp4
+2024-12-17 22:10:42,648 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json
+2024-12-17 22:10:42,648 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/0_eval_0_f8fef5727eff9ff71143.mp4
+2024-12-17 22:10:42,649 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos
+2024-12-17 22:10:42,649 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media
+2024-12-17 22:10:42,649 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval
+2024-12-17 22:10:42,911 INFO    wandb-upload_0:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/e8hb5f9e-media/videos/eval/0_eval_0_f8fef5727eff9ff71143.mp4
+2024-12-17 22:10:43,090 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:43,349 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:10:43,649 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:44,090 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:45,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:46,091 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:47,090 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:47,650 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:48,091 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:48,458 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:10:49,091 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:49,651 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:50,091 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:51,091 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:52,091 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:52,652 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0
+2024-12-17 22:10:53,091 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:53,652 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:53,973 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:10:54,091 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:54,729 DEBUG   SystemMonitor:82813 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-12-17 22:10:54,729 DEBUG   SenderThread:82813 [sender.py:send():379] send: stats
+2024-12-17 22:10:55,083 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:10:55,083 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:10:55,125 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:55,653 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:56,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:57,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:57,653 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:10:58,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:59,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:10:59,868 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:11:00,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:01,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:02,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:03,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:04,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:05,085 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:11:05,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:06,093 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:07,093 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:08,093 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:09,093 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:10,083 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:11:10,083 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:11:10,129 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:10,240 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:11:11,093 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:12,093 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:13,093 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:13,659 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:14,094 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:15,094 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:15,569 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:11:15,659 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:16,096 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:17,095 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:17,660 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:18,097 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:19,096 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:19,661 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:20,096 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:21,085 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:11:21,096 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:21,662 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:22,096 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:23,096 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:23,662 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:23,747 DEBUG   SenderThread:82813 [sender.py:send():379] send: files
+2024-12-17 22:11:23,747 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_8a67326b5dfefa361130.mp4 with policy now
+2024-12-17 22:11:23,750 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 22:11:23,751 DEBUG   SenderThread:82813 [sender.py:send():379] send: history
+2024-12-17 22:11:23,751 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:11:23,751 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:11:24,004 INFO    wandb-upload_1:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/wz2w0qdj-media/videos/eval/1_eval_1_8a67326b5dfefa361130.mp4
+2024-12-17 22:11:24,097 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:24,663 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json
+2024-12-17 22:11:24,663 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/1_eval_1_8a67326b5dfefa361130.mp4
+2024-12-17 22:11:24,664 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval
+2024-12-17 22:11:24,731 DEBUG   SenderThread:82813 [sender.py:send():379] send: stats
+2024-12-17 22:11:24,963 INFO    wandb-upload_0:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/tplhjs43-media/videos/eval/1_eval_1_8a67326b5dfefa361130.mp4
+2024-12-17 22:11:25,085 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:11:25,085 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:11:25,129 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:25,663 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:26,101 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:26,187 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:11:26,664 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:27,102 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:28,100 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:29,105 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:29,664 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:30,102 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:31,114 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:31,665 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:32,086 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:11:32,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:33,103 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:34,105 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:35,104 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:35,666 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0
+2024-12-17 22:11:35,910 INFO    wandb-upload_1:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/f5kpna1a-events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0
+2024-12-17 22:11:36,103 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:36,489 DEBUG   SenderThread:82813 [sender.py:send():379] send: history
+2024-12-17 22:11:36,489 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:11:36,489 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:11:36,667 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json
+2024-12-17 22:11:37,110 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:37,490 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:11:37,667 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:38,110 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:39,107 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:40,096 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:11:40,096 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:11:40,137 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:41,107 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:41,668 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:42,108 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:43,108 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:43,385 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:11:43,669 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:44,108 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:45,108 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:46,108 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:47,108 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:47,671 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:48,108 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:49,108 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:49,329 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:11:49,671 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:50,108 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:51,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:52,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:53,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:53,673 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:54,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:54,730 DEBUG   SenderThread:82813 [sender.py:send():379] send: stats
+2024-12-17 22:11:54,731 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:11:55,086 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:11:55,087 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:11:55,129 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:55,673 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:11:56,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:57,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:58,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:11:59,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:00,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:00,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:01,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:02,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:03,109 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:04,110 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:05,088 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:05,110 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:06,110 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:07,110 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:08,110 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:08,911 DEBUG   SenderThread:82813 [sender.py:send():379] send: files
+2024-12-17 22:12:08,911 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_4735c6188bf9f36869cb.mp4 with policy now
+2024-12-17 22:12:08,914 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 22:12:08,914 DEBUG   SenderThread:82813 [sender.py:send():379] send: history
+2024-12-17 22:12:08,914 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:12:08,914 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:12:09,118 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:09,174 INFO    wandb-upload_0:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/b0oeewff-media/videos/eval/2_eval_2_4735c6188bf9f36869cb.mp4
+2024-12-17 22:12:09,678 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:12:09,678 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json
+2024-12-17 22:12:09,678 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/2_eval_2_4735c6188bf9f36869cb.mp4
+2024-12-17 22:12:09,678 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval
+2024-12-17 22:12:10,003 INFO    wandb-upload_1:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/w5y0lzh2-media/videos/eval/2_eval_2_4735c6188bf9f36869cb.mp4
+2024-12-17 22:12:10,090 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:12:10,091 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:12:10,133 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:10,234 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:11,114 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:11,678 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:12:12,123 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:13,120 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:13,679 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:12:14,133 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:15,125 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:15,339 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:16,131 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:16,680 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0
+2024-12-17 22:12:17,134 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:17,324 DEBUG   SenderThread:82813 [sender.py:send():379] send: history
+2024-12-17 22:12:17,324 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:12:17,325 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:12:17,681 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:12:17,681 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json
+2024-12-17 22:12:18,135 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:19,145 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:19,681 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:12:20,141 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:20,399 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:21,142 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:21,682 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:12:22,137 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:23,138 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:23,683 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:12:24,138 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:24,731 DEBUG   SenderThread:82813 [sender.py:send():379] send: stats
+2024-12-17 22:12:25,090 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:12:25,090 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:12:25,138 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:26,139 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:26,223 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:27,140 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:28,141 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:29,141 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:30,142 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:31,141 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:32,093 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:32,142 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:33,144 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:34,143 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:35,141 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:36,142 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:37,093 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:37,143 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:38,143 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:39,144 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:40,089 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:12:40,089 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:12:40,144 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:41,143 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:42,145 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:42,184 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:43,145 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:44,144 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:45,144 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:46,144 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:47,144 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:48,094 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:48,145 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:49,145 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:50,173 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:50,390 DEBUG   SenderThread:82813 [sender.py:send():379] send: files
+2024-12-17 22:12:50,390 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_a4af24fd7d85bfc8c826.mp4 with policy now
+2024-12-17 22:12:50,394 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 22:12:50,395 DEBUG   SenderThread:82813 [sender.py:send():379] send: history
+2024-12-17 22:12:50,395 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:12:50,396 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:12:50,642 INFO    wandb-upload_0:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/o2w5qpu6-media/videos/eval/3_eval_3_a4af24fd7d85bfc8c826.mp4
+2024-12-17 22:12:50,691 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json
+2024-12-17 22:12:50,691 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/3_eval_3_a4af24fd7d85bfc8c826.mp4
+2024-12-17 22:12:50,692 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval
+2024-12-17 22:12:50,960 INFO    wandb-upload_1:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/69ml2jnp-media/videos/eval/3_eval_3_a4af24fd7d85bfc8c826.mp4
+2024-12-17 22:12:51,162 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:51,692 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:12:52,156 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:53,153 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:53,406 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:54,158 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:54,731 DEBUG   SenderThread:82813 [sender.py:send():379] send: stats
+2024-12-17 22:12:55,092 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:12:55,092 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:12:55,159 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:55,693 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:12:56,162 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:57,152 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:57,694 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:12:58,156 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:12:58,873 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:12:59,163 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:00,165 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:01,162 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:02,159 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:03,169 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:04,104 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:13:04,167 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:05,163 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:06,167 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:07,173 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:08,174 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:09,175 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:09,384 DEBUG   SenderThread:82813 [sender.py:send():379] send: history
+2024-12-17 22:13:09,384 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:13:09,384 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:13:09,384 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:13:09,697 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:09,698 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json
+2024-12-17 22:13:09,698 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0
+2024-12-17 22:13:09,954 INFO    wandb-upload_0:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/5zmrcuk8-events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0
+2024-12-17 22:13:10,097 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:13:10,098 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:13:10,176 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:11,181 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:11,698 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:12,180 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:13,177 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:13,699 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:14,178 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:14,672 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:13:15,178 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:15,700 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:16,178 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:17,178 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:18,178 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:19,178 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:20,098 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:13:20,178 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:21,178 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:22,179 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:23,179 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:24,179 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:24,732 DEBUG   SenderThread:82813 [sender.py:send():379] send: stats
+2024-12-17 22:13:25,089 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:13:25,090 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:13:25,179 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:25,221 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:13:25,703 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:26,179 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:27,179 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:28,180 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:29,180 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:30,180 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:31,099 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:13:31,180 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:32,180 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:33,180 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:34,180 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:35,181 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:36,181 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:37,099 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:13:37,183 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:38,181 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:39,184 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:40,090 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: stop_status
+2024-12-17 22:13:40,090 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: stop_status
+2024-12-17 22:13:40,183 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:41,195 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:41,238 DEBUG   SenderThread:82813 [sender.py:send():379] send: files
+2024-12-17 22:13:41,238 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_ccefe4eb2a70fd4856d6.mp4 with policy now
+2024-12-17 22:13:41,285 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: partial_history
+2024-12-17 22:13:41,285 DEBUG   SenderThread:82813 [sender.py:send():379] send: history
+2024-12-17 22:13:41,286 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:13:41,286 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:13:41,498 INFO    wandb-upload_1:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/chvo9qog-media/videos/eval/4_eval_4_ccefe4eb2a70fd4856d6.mp4
+2024-12-17 22:13:41,708 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:41,708 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json
+2024-12-17 22:13:41,709 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/4_eval_4_ccefe4eb2a70fd4856d6.mp4
+2024-12-17 22:13:41,709 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval
+2024-12-17 22:13:41,976 INFO    wandb-upload_0:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/b9kgywxw-media/videos/eval/4_eval_4_ccefe4eb2a70fd4856d6.mp4
+2024-12-17 22:13:42,187 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:42,286 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:13:43,186 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:43,709 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:44,195 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:45,198 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:45,710 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:46,195 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:47,198 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:47,710 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:47,830 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:13:48,207 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:49,219 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:13:49,711 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:49,957 DEBUG   SenderThread:82813 [sender.py:send():379] send: exit
+2024-12-17 22:13:49,957 INFO    SenderThread:82813 [sender.py:send_exit():586] handling exit code: 0
+2024-12-17 22:13:49,957 INFO    SenderThread:82813 [sender.py:send_exit():588] handling runtime: 235
+2024-12-17 22:13:49,958 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:13:49,958 INFO    SenderThread:82813 [sender.py:send_exit():594] send defer
+2024-12-17 22:13:49,958 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:13:49,958 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 0
+2024-12-17 22:13:49,958 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:13:49,958 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 0
+2024-12-17 22:13:49,958 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 1
+2024-12-17 22:13:49,958 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:13:49,958 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 1
+2024-12-17 22:13:49,959 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:13:49,959 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 1
+2024-12-17 22:13:49,959 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 2
+2024-12-17 22:13:49,959 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:13:49,959 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 2
+2024-12-17 22:13:49,959 INFO    HandlerThread:82813 [system_monitor.py:finish():203] Stopping system monitor
+2024-12-17 22:13:49,959 DEBUG   SystemMonitor:82813 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-12-17 22:13:49,959 INFO    HandlerThread:82813 [interfaces.py:finish():200] Joined cpu monitor
+2024-12-17 22:13:49,959 DEBUG   SystemMonitor:82813 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-12-17 22:13:49,959 INFO    HandlerThread:82813 [interfaces.py:finish():200] Joined disk monitor
+2024-12-17 22:13:50,007 INFO    HandlerThread:82813 [interfaces.py:finish():200] Joined gpu monitor
+2024-12-17 22:13:50,007 INFO    HandlerThread:82813 [interfaces.py:finish():200] Joined memory monitor
+2024-12-17 22:13:50,007 INFO    HandlerThread:82813 [interfaces.py:finish():200] Joined network monitor
+2024-12-17 22:13:50,008 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:13:50,008 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 2
+2024-12-17 22:13:50,008 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 3
+2024-12-17 22:13:50,008 DEBUG   SenderThread:82813 [sender.py:send():379] send: stats
+2024-12-17 22:13:50,008 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:13:50,008 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 3
+2024-12-17 22:13:50,008 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:13:50,008 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 3
+2024-12-17 22:13:50,008 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 4
+2024-12-17 22:13:50,008 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:13:50,008 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 4
+2024-12-17 22:13:50,711 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json
+2024-12-17 22:13:50,712 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0
+2024-12-17 22:13:51,712 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:56,010 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:13:56,011 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:13:56,011 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:13:56,011 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: keepalive
+2024-12-17 22:13:56,011 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 4
+2024-12-17 22:13:56,011 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 5
+2024-12-17 22:13:56,011 DEBUG   SenderThread:82813 [sender.py:send():379] send: history
+2024-12-17 22:13:56,011 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:13:56,011 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:13:56,012 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 5
+2024-12-17 22:13:56,013 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:13:56,013 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:13:56,013 DEBUG   SenderThread:82813 [sender.py:send():379] send: history
+2024-12-17 22:13:56,013 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: summary_record
+2024-12-17 22:13:56,017 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:13:56,017 DEBUG   SenderThread:82813 [sender.py:send():379] send: summary
+2024-12-17 22:13:56,017 INFO    SenderThread:82813 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-12-17 22:13:56,017 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:13:56,017 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 5
+2024-12-17 22:13:56,018 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 6
+2024-12-17 22:13:56,018 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:13:56,018 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 6
+2024-12-17 22:13:56,018 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:13:56,018 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 6
+2024-12-17 22:13:56,018 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 7
+2024-12-17 22:13:56,018 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: status_report
+2024-12-17 22:13:56,018 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:13:56,018 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 7
+2024-12-17 22:13:56,018 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:13:56,018 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 7
+2024-12-17 22:13:56,713 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json
+2024-12-17 22:13:56,958 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:13:57,893 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 8
+2024-12-17 22:13:57,897 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:13:57,897 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:13:57,898 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 8
+2024-12-17 22:13:57,898 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:13:57,898 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 8
+2024-12-17 22:13:57,898 INFO    SenderThread:82813 [job_builder.py:build():440] Attempting to build job artifact
+2024-12-17 22:13:57,900 INFO    SenderThread:82813 [job_builder.py:_get_source_type():569] is repo sourced job
+2024-12-17 22:13:57,914 INFO    SenderThread:82813 [job_builder.py:build():545] adding wandb-job metadata file
+2024-12-17 22:13:57,924 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 9
+2024-12-17 22:13:57,924 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:13:57,924 DEBUG   SenderThread:82813 [sender.py:send():379] send: artifact
+2024-12-17 22:13:57,925 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 9
+2024-12-17 22:13:57,959 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:13:58,703 INFO    wandb-upload_0:82813 [upload_job.py:push():85] Skipped uploading /tmp/tmpzprnyq1w/wandb-job.json
+2024-12-17 22:13:58,703 INFO    wandb-upload_2:82813 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp9f2r2ubh
+2024-12-17 22:13:58,714 INFO    Thread-12 :82813 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:58,826 INFO    wandb-upload_1:82813 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmptk53d0ug
+2024-12-17 22:13:59,393 INFO    SenderThread:82813 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM4MzM4ODU2OA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM4MzM3MTUyNg==', 'versionIndex': 382}}}
+2024-12-17 22:13:59,393 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:13:59,393 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 9
+2024-12-17 22:13:59,393 INFO    SenderThread:82813 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-12-17 22:13:59,715 INFO    SenderThread:82813 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files
+2024-12-17 22:13:59,715 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/config.yaml config.yaml
+2024-12-17 22:13:59,715 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json wandb-summary.json
+2024-12-17 22:13:59,715 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/requirements.txt requirements.txt
+2024-12-17 22:13:59,718 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-metadata.json wandb-metadata.json
+2024-12-17 22:13:59,718 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0 events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0
+2024-12-17 22:13:59,719 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log output.log
+2024-12-17 22:13:59,720 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/diff.patch diff.patch
+2024-12-17 22:13:59,720 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/code/FISH/eval_robot.py code/FISH/eval_robot.py
+2024-12-17 22:13:59,720 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/1_eval_1_8a67326b5dfefa361130.mp4 media/videos/eval/1_eval_1_8a67326b5dfefa361130.mp4
+2024-12-17 22:13:59,720 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/0_eval_0_f8fef5727eff9ff71143.mp4 media/videos/eval/0_eval_0_f8fef5727eff9ff71143.mp4
+2024-12-17 22:13:59,720 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/4_eval_4_ccefe4eb2a70fd4856d6.mp4 media/videos/eval/4_eval_4_ccefe4eb2a70fd4856d6.mp4
+2024-12-17 22:13:59,720 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/3_eval_3_a4af24fd7d85bfc8c826.mp4 media/videos/eval/3_eval_3_a4af24fd7d85bfc8c826.mp4
+2024-12-17 22:13:59,722 INFO    SenderThread:82813 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/media/videos/eval/2_eval_2_4735c6188bf9f36869cb.mp4 media/videos/eval/2_eval_2_4735c6188bf9f36869cb.mp4
+2024-12-17 22:13:59,722 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 10
+2024-12-17 22:13:59,724 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:13:59,724 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:13:59,724 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 10
+2024-12-17 22:13:59,724 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:13:59,724 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 10
+2024-12-17 22:13:59,724 INFO    SenderThread:82813 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 22:13:59,900 INFO    wandb-upload_1:82813 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/requirements.txt
+2024-12-17 22:13:59,920 INFO    wandb-upload_0:82813 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/config.yaml
+2024-12-17 22:13:59,932 INFO    wandb-upload_2:82813 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/wandb-summary.json
+2024-12-17 22:13:59,943 INFO    wandb-upload_4:82813 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/files/output.log
+2024-12-17 22:13:59,960 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:13:59,960 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:14:00,035 INFO    wandb-upload_3:82813 [upload_job.py:push():130] Uploaded file /tmp/tmpyykw5wcjwandb/gqgju0j7-events.out.tfevents.1734491395.leonmkim-ROG-Strix-G15CS-G15CS.82633.0
+2024-12-17 22:14:00,236 INFO    Thread-11 (_thread_body):82813 [sender.py:transition_state():614] send defer: 11
+2024-12-17 22:14:00,236 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:14:00,236 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 11
+2024-12-17 22:14:00,236 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:14:00,236 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 11
+2024-12-17 22:14:00,236 INFO    SenderThread:82813 [file_pusher.py:join():175] waiting for file pusher
+2024-12-17 22:14:00,237 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 12
+2024-12-17 22:14:00,237 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:14:00,237 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 12
+2024-12-17 22:14:00,237 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:14:00,237 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 12
+2024-12-17 22:14:00,238 INFO    SenderThread:82813 [file_stream.py:finish():601] file stream finish called
+2024-12-17 22:14:00,435 INFO    SenderThread:82813 [file_stream.py:finish():605] file stream finish is done
+2024-12-17 22:14:00,436 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 13
+2024-12-17 22:14:00,436 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:14:00,436 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 13
+2024-12-17 22:14:00,436 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:14:00,436 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 13
+2024-12-17 22:14:00,436 INFO    SenderThread:82813 [sender.py:transition_state():614] send defer: 14
+2024-12-17 22:14:00,436 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: defer
+2024-12-17 22:14:00,436 INFO    HandlerThread:82813 [handler.py:handle_request_defer():184] handle defer: 14
+2024-12-17 22:14:00,437 DEBUG   SenderThread:82813 [sender.py:send():379] send: final
+2024-12-17 22:14:00,437 DEBUG   SenderThread:82813 [sender.py:send():379] send: footer
+2024-12-17 22:14:00,437 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: defer
+2024-12-17 22:14:00,437 INFO    SenderThread:82813 [sender.py:send_request_defer():610] handle sender defer: 14
+2024-12-17 22:14:00,437 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:14:00,437 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:14:00,438 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: server_info
+2024-12-17 22:14:00,438 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: server_info
+2024-12-17 22:14:00,438 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: poll_exit
+2024-12-17 22:14:00,440 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: get_summary
+2024-12-17 22:14:00,440 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: sampled_history
+2024-12-17 22:14:00,440 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: internal_messages
+2024-12-17 22:14:00,521 DEBUG   SenderThread:82813 [sender.py:send_request():406] send_request: poll_exit
+2024-12-17 22:14:00,521 INFO    MainThread:82813 [wandb_run.py:_footer_history_summary_info():4016] rendering history
+2024-12-17 22:14:00,522 INFO    MainThread:82813 [wandb_run.py:_footer_history_summary_info():4048] rendering summary
+2024-12-17 22:14:00,522 INFO    MainThread:82813 [wandb_run.py:_footer_sync_info():3975] logging synced files
+2024-12-17 22:14:00,522 DEBUG   HandlerThread:82813 [handler.py:handle_request():158] handle_request: shutdown
+2024-12-17 22:14:00,522 INFO    HandlerThread:82813 [handler.py:finish():882] shutting down handler
+2024-12-17 22:14:01,440 INFO    WriterThread:82813 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/run-7lp0oxke.wandb
+2024-12-17 22:14:01,521 INFO    SenderThread:82813 [sender.py:finish():1615] shutting down sender
+2024-12-17 22:14:01,521 INFO    SenderThread:82813 [file_pusher.py:finish():169] shutting down file pusher
+2024-12-17 22:14:01,521 INFO    SenderThread:82813 [file_pusher.py:join():175] waiting for file pusher

220948/wandb/run-20241217_220954-7lp0oxke/logs/debug.log

@@ -0,0 +1,31 @@
+2024-12-17 22:09:54,388 INFO    MainThread:82633 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5
+2024-12-17 22:09:54,388 INFO    MainThread:82633 [wandb_setup.py:_flush():76] Configure stats pid to 82633
+2024-12-17 22:09:54,388 INFO    MainThread:82633 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings
+2024-12-17 22:09:54,388 INFO    MainThread:82633 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/settings
+2024-12-17 22:09:54,388 INFO    MainThread:82633 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-12-17 22:09:54,388 INFO    MainThread:82633 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-12-17 22:09:54,388 INFO    MainThread:82633 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'}
+2024-12-17 22:09:54,389 INFO    MainThread:82633 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-12-17 22:09:54,389 INFO    MainThread:82633 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/logs/debug.log
+2024-12-17 22:09:54,389 INFO    MainThread:82633 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/wandb/run-20241217_220954-7lp0oxke/logs/debug-internal.log
+2024-12-17 22:09:54,389 INFO    MainThread:82633 [wandb_init.py:init():569] calling init triggers
+2024-12-17 22:09:54,389 INFO    MainThread:82633 [wandb_init.py:init():576] wandb.init called with sweep_config: {}
+config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '510_0', 'wandb_notes': '510_0_', 'eval': True, 'true_action_history': False, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': False, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['modelsys_twodim'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/510_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948'}
+2024-12-17 22:09:54,389 INFO    MainThread:82633 [wandb_init.py:init():619] starting backend
+2024-12-17 22:09:54,389 INFO    MainThread:82633 [wandb_init.py:init():623] setting up manager
+2024-12-17 22:09:54,392 INFO    MainThread:82633 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-17 22:09:54,394 INFO    MainThread:82633 [wandb_init.py:init():631] backend started and connected
+2024-12-17 22:09:54,409 INFO    MainThread:82633 [wandb_init.py:init():720] updated telemetry
+2024-12-17 22:09:54,416 INFO    MainThread:82633 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2024-12-17 22:09:54,649 INFO    MainThread:82633 [wandb_run.py:_on_init():2435] communicating current version
+2024-12-17 22:09:54,716 INFO    MainThread:82633 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-12-17 22:09:54,716 INFO    MainThread:82633 [wandb_init.py:init():804] starting run threads in backend
+2024-12-17 22:09:55,083 INFO    MainThread:82633 [wandb_run.py:_console_start():2413] atexit reg
+2024-12-17 22:09:55,083 INFO    MainThread:82633 [wandb_run.py:_redirect():2255] redirect: wrap_raw
+2024-12-17 22:09:55,083 INFO    MainThread:82633 [wandb_run.py:_redirect():2320] Wrapping output streams.
+2024-12-17 22:09:55,083 INFO    MainThread:82633 [wandb_run.py:_redirect():2345] Redirects installed.
+2024-12-17 22:09:55,086 INFO    MainThread:82633 [wandb_init.py:init():847] run started, returning control to user process
+2024-12-17 22:09:55,086 INFO    MainThread:82633 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/510_0/220948/tb, True
+2024-12-17 22:10:00,474 INFO    MainThread:82633 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'}
+2024-12-17 22:14:01,524 WARNING MsgRouterThr:82633 [router.py:message_loop():77] message_loop has been closed

config.yaml

@@ -0,0 +1,546 @@
+root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data
+replay_buffer_size: 150000
+replay_buffer_num_workers: 2
+nstep: 3
+batch_size: 128
+seed: 0
+dataset_shuffle_seed: 0
+device: cuda
+save_video: true
+save_train_video: true
+use_tb: true
+use_wandb: true
+wandb_run_id: '510_0'
+wandb_notes: '510_0_'
+eval: false
+true_action_history: false
+process_contact_features: ${eval}
+obs_type: pixels
+use_color: true
+use_depth: true
+use_masks: false
+mask_list:
+- EE_obj_mask
+mask_representation: channels
+crop_hw:
+- 144
+- 144
+crop_down_offset: 48
+color_crop_type: null
+depth_crop_type: null
+segmask_crop_type: null
+add_crop_binary_mask: false
+add_coord_conv_map: false
+use_context_color: false
+use_context_depth: false
+use_context_segmask: false
+context_color_crop_type: null
+context_depth_crop_type: null
+context_segmask_crop_type: null
+context_add_crop_binary_mask: false
+context_add_coord_conv_map: false
+use_contact_map: false
+use_sdf_maps: false
+use_normals_maps: false
+which_objects: both
+max_contact_prob: 0.1
+max_depth: 2.0
+grasped_dtc_max_value: 0.2
+env_dtc_max_value: 0.4
+grasped_normals_mask_max_dtc_value: 0.2
+env_normals_mask_max_dtc_value: 0.4
+clamp_dtc: true
+dtc_adaptive_normalization: false
+mask_normals_within_sdf: true
+adaptive_normals_mask: true
+learnable_contact_preprocess_params: true
+contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9
+contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt
+encoder_type: small
+debug_timestamps: false
+open_loop: false
+action_trajectories: true
+stop_after_action: false
+interpolation_frequency: 25
+policy_frequency: 5
+wait_for_new_camera_frames: true
+baseline: false
+train_demo_idxs_list_or_num: -1
+log_train_every_steps: 25
+name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo}
+store_dataset_in_memory: false
+expert_dataset: ${expert_dataset_dirpath}/demos.zarr
+action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz',
+  'action'}
+semantic_demo_grouping_name: semantic_demo_grouping.yaml
+semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name}
+include_groups_list:
+- modelsys_twodim
+expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml
+name_of_valid_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act
+valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr
+valid_demo_idxs_list_or_num: null
+val_num_groups: 0
+load_bc: ${agent.load_checkpoint}
+checkpoint_epoch_list:
+- 99
+- 199
+- 299
+- 399
+- 499
+- 599
+- 699
+- 799
+- 899
+- 999
+- 1249
+- 1499
+- 1749
+- 1999
+- 2999
+- 3999
+- 4999
+- 5999
+- 6999
+- 7999
+- 8999
+- 9999
+snapshot_root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data/FISH
+save_snapshot: true
+save_last_snapshot: true
+save_snapshot_when_done: true
+top_k_checkpoints: 5
+save_snapshot_link_to_weights_dir: deprecated
+bc_regularize: false
+bc_weight_type: qfilter
+experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id}
+agent:
+  _target_: agent.diffusion_policy.DiffusionPolicyAgent
+  name: diffusion_policy
+  load_checkpoint: ${eval}
+  device: ${device}
+  n_obs_steps: ${.config.policy_cfg.n_obs_steps}
+  suite_name: ${suite.name}
+  obs_type: ${obs_type}
+  enable_arm: ${eval}
+  enable_camera: ${eval}
+  use_tb: ${use_tb}
+  desired_image_shape:
+  - 13
+  - 180
+  - 240
+  orig_cam_shape:
+  - 3
+  - 240
+  - 320
+  config:
+    _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+    compile: false
+    device: ${device}
+    cam_resize_shape: ${agent.desired_image_shape}
+    orig_cam_shape: ${agent.orig_cam_shape}
+    policy_frequency: ${policy_frequency}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_cfg:
+      _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+      n_obs_steps: 1
+      horizon: 36
+      n_action_steps: ${agent.config.policy_cfg.horizon}
+      output_shapes:
+        action:
+        - 7
+      input_normalization_modes:
+        observation.image: mean_std
+        observation.state: min_max
+        observation.action_history: min_max
+      output_normalization_modes:
+        action: min_max
+      vision_backbone: resnet18
+      pretrained_backbone_weights: null
+      transforms:
+      - _target_: torchaug.transforms.RandomAffine
+        degrees:
+        - -5
+        - 5
+        translate:
+        - 0.05
+        - 0.05
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      - _target_: torchaug.transforms.RandomColorJitter
+        brightness: 0.3
+        contrast: 0.4
+        saturation: 0.5
+        hue: 0.08
+        batch_transform: true
+        num_chunks: -1
+        batch_inplace: true
+      use_group_norm: true
+      spatial_softmax_num_keypoints: 32
+      action_history_encoder_config:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+        in_channels: 7
+        out_channels: 32
+        history_length: 6
+        kernel_size: ${agent.config.policy_cfg.kernel_size}
+        downsample_kernel_size: 3
+        downsample_stride: 2
+        downsample_padding: 1
+      down_dims:
+      - 256
+      - 512
+      - 1024
+      kernel_size: 5
+      n_groups: 8
+      diffusion_step_embed_dim: 128
+      use_film_scale_modulation: true
+      noise_scheduler_type: DDIM
+      beta_schedule: squaredcos_cap_v2
+      beta_start: 0.0001
+      beta_end: 0.02
+      prediction_type: epsilon
+      clip_sample: true
+      clip_sample_range: 1.0
+      num_train_timesteps: 50
+      num_inference_steps: 10
+      do_mask_loss_for_padding: false
+      input_shapes:
+        observation.image:
+        - 13
+        - 180
+        - 240
+        context_observation.image:
+        - 13
+        - 180
+        - 240
+        observation.state:
+        - 8
+        observation.action_history:
+        - 7
+    train_cfg:
+      _target_: utils.TrainConfig
+      lr: 0.0001
+      lr_scheduler: cosine
+      lr_warmup_steps: 500
+      adam_betas:
+      - 0.95
+      - 0.999
+      adam_eps: 1.0e-08
+      adam_weight_decay: 1.0e-06
+      grad_clip_norm: 10
+      offline_steps: ${num_train_frames_diffusion}
+      use_amp: true
+    observation_cfg:
+      _target_: agent.encoder.VisualFeatureSet
+      use_depth: ${use_depth}
+      use_color: ${use_color}
+      mask_input_dict:
+        _target_: agent.encoder.MaskInputDict
+        enable: ${use_masks}
+        representation: ${mask_representation}
+        mask_list: ${mask_list}
+      crop_input_config:
+        _target_: agent.encoder.CropInputConfig
+        color_crop_type: ${color_crop_type}
+        depth_crop_type: ${depth_crop_type}
+        segmask_crop_type: ${segmask_crop_type}
+        crop_hw: ${crop_hw}
+        crop_down_offset: ${crop_down_offset}
+        add_crop_binary_mask: ${add_crop_binary_mask}
+        add_coord_conv_map: ${add_coord_conv_map}
+      context_input_config:
+        _target_: agent.encoder.ContextInputConfig
+        use_color: ${use_context_color}
+        use_depth: ${use_context_depth}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_context_segmask}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${context_color_crop_type}
+          depth_crop_type: ${context_depth_crop_type}
+          segmask_crop_type: ${context_segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${context_add_crop_binary_mask}
+          add_coord_conv_map: ${context_add_coord_conv_map}
+      mask_soft_approx_scheduler_config:
+        _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+        num_steps: 40000
+        initial_value: 10.0
+        final_value: 1000.0
+        interpolation_scheme: cosine
+      use_contact_map: ${use_contact_map}
+      use_sdf_maps: ${use_sdf_maps}
+      use_normals_maps: ${use_normals_maps}
+      which_objects: ${which_objects}
+      grasped_dtc_max_value: ${grasped_dtc_max_value}
+      env_dtc_max_value: ${env_dtc_max_value}
+      grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+      env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+      clamp_dtc: ${clamp_dtc}
+      max_contact_prob: ${max_contact_prob}
+      mask_normals_within_sdf: ${mask_normals_within_sdf}
+      dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+      adaptive_normals_mask: ${adaptive_normals_mask}
+      max_depth: ${max_depth}
+      image_shape: ${agent.desired_image_shape}
+      learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+      learning_rate: 0.0001
+      weight_decay: 0.0
+      contact_model_name: ${contact_model_name}
+      zero_centered: false
+suite:
+  suite: frankagym
+  name: frankagym
+  frame_stack: ${agent.n_obs_steps}
+  action_repeat: 1
+  discount: 0.99
+  hidden_dim: 1024
+  num_train_frames: 1000000
+  num_seed_frames: 0
+  num_train_epochs: 15000
+  validate_every_epochs: 250
+  validate_diffusion_on_action_loss_every_epochs: 250
+  train_eval_diffusion_on_action_loss_every_epochs: 250
+  check_topk_every_epochs: 250
+  save_snapshot_every_epochs: 1500
+  eval_every_frames: 2000
+  num_eval_episodes: 5
+  save_snapshot: true
+  wait_for_user_to_start_episode: true
+  task_make_fn:
+    _target_: suite.frankagym.make
+    name: ${task_name}
+    height: 240
+    width: 320
+    frame_stack: ${suite.frame_stack}
+    action_repeat: ${suite.action_repeat}
+    seed: ${seed}
+    enable_arm: ${agent.enable_arm}
+    enable_gripper: ${enable_gripper}
+    start_with_gripper_open: ${start_with_gripper_open}
+    enable_camera: ${agent.enable_camera}
+    path_to_depth_extrinsics: ${path_to_depth_extrinsics}
+    contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path}
+    x_limit: ${x_limit}
+    y_limit: ${y_limit}
+    z_limit: ${z_limit}
+    device: ${device}
+    interpolation_frequency: ${interpolation_frequency}
+    policy_frequency: ${policy_frequency}
+    debug_timestamps: ${debug_timestamps}
+    stop_after_action: ${stop_after_action}
+    open_loop: ${open_loop}
+    wait_for_new_camera_frames: ${wait_for_new_camera_frames}
+    action_key: ${action_key}
+    action_trajectory_horizon: ${agent.config.policy_cfg.horizon}
+    action_trajectories: ${action_trajectories}
+    path_to_zarr_dataset: ${expert_dataset}
+    agent_policy_cfg:
+      _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig
+      compile: false
+      device: ${device}
+      cam_resize_shape: ${agent.desired_image_shape}
+      orig_cam_shape: ${agent.orig_cam_shape}
+      policy_frequency: ${policy_frequency}
+      interpolation_frequency: ${interpolation_frequency}
+      policy_cfg:
+        _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig
+        n_obs_steps: 1
+        horizon: 36
+        n_action_steps: ${agent.config.policy_cfg.horizon}
+        output_shapes:
+          action:
+          - 7
+        input_normalization_modes:
+          observation.image: mean_std
+          observation.state: min_max
+          observation.action_history: min_max
+        output_normalization_modes:
+          action: min_max
+        vision_backbone: resnet18
+        pretrained_backbone_weights: null
+        transforms:
+        - _target_: torchaug.transforms.RandomAffine
+          degrees:
+          - -5
+          - 5
+          translate:
+          - 0.05
+          - 0.05
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        - _target_: torchaug.transforms.RandomColorJitter
+          brightness: 0.3
+          contrast: 0.4
+          saturation: 0.5
+          hue: 0.08
+          batch_transform: true
+          num_chunks: -1
+          batch_inplace: true
+        use_group_norm: true
+        spatial_softmax_num_keypoints: 32
+        action_history_encoder_config:
+          _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig
+          in_channels: 7
+          out_channels: 32
+          history_length: 6
+          kernel_size: ${agent.config.policy_cfg.kernel_size}
+          downsample_kernel_size: 3
+          downsample_stride: 2
+          downsample_padding: 1
+        down_dims:
+        - 256
+        - 512
+        - 1024
+        kernel_size: 5
+        n_groups: 8
+        diffusion_step_embed_dim: 128
+        use_film_scale_modulation: true
+        noise_scheduler_type: DDIM
+        beta_schedule: squaredcos_cap_v2
+        beta_start: 0.0001
+        beta_end: 0.02
+        prediction_type: epsilon
+        clip_sample: true
+        clip_sample_range: 1.0
+        num_train_timesteps: 50
+        num_inference_steps: 10
+        do_mask_loss_for_padding: false
+        input_shapes:
+          observation.image:
+          - 13
+          - 180
+          - 240
+          context_observation.image:
+          - 13
+          - 180
+          - 240
+          observation.state:
+          - 8
+          observation.action_history:
+          - 7
+      train_cfg:
+        _target_: utils.TrainConfig
+        lr: 0.0001
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        adam_betas:
+        - 0.95
+        - 0.999
+        adam_eps: 1.0e-08
+        adam_weight_decay: 1.0e-06
+        grad_clip_norm: 10
+        offline_steps: ${num_train_frames_diffusion}
+        use_amp: true
+      observation_cfg:
+        _target_: agent.encoder.VisualFeatureSet
+        use_depth: ${use_depth}
+        use_color: ${use_color}
+        mask_input_dict:
+          _target_: agent.encoder.MaskInputDict
+          enable: ${use_masks}
+          representation: ${mask_representation}
+          mask_list: ${mask_list}
+        crop_input_config:
+          _target_: agent.encoder.CropInputConfig
+          color_crop_type: ${color_crop_type}
+          depth_crop_type: ${depth_crop_type}
+          segmask_crop_type: ${segmask_crop_type}
+          crop_hw: ${crop_hw}
+          crop_down_offset: ${crop_down_offset}
+          add_crop_binary_mask: ${add_crop_binary_mask}
+          add_coord_conv_map: ${add_coord_conv_map}
+        context_input_config:
+          _target_: agent.encoder.ContextInputConfig
+          use_color: ${use_context_color}
+          use_depth: ${use_context_depth}
+          mask_input_dict:
+            _target_: agent.encoder.MaskInputDict
+            enable: ${use_context_segmask}
+            representation: ${mask_representation}
+            mask_list: ${mask_list}
+          crop_input_config:
+            _target_: agent.encoder.CropInputConfig
+            color_crop_type: ${context_color_crop_type}
+            depth_crop_type: ${context_depth_crop_type}
+            segmask_crop_type: ${context_segmask_crop_type}
+            crop_hw: ${crop_hw}
+            crop_down_offset: ${crop_down_offset}
+            add_crop_binary_mask: ${context_add_crop_binary_mask}
+            add_coord_conv_map: ${context_add_coord_conv_map}
+        mask_soft_approx_scheduler_config:
+          _target_: agent.encoder.MaskSoftApproxSchedulerConfig
+          num_steps: 40000
+          initial_value: 10.0
+          final_value: 1000.0
+          interpolation_scheme: cosine
+        use_contact_map: ${use_contact_map}
+        use_sdf_maps: ${use_sdf_maps}
+        use_normals_maps: ${use_normals_maps}
+        which_objects: ${which_objects}
+        grasped_dtc_max_value: ${grasped_dtc_max_value}
+        env_dtc_max_value: ${env_dtc_max_value}
+        grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value}
+        env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value}
+        clamp_dtc: ${clamp_dtc}
+        max_contact_prob: ${max_contact_prob}
+        mask_normals_within_sdf: ${mask_normals_within_sdf}
+        dtc_adaptive_normalization: ${dtc_adaptive_normalization}
+        adaptive_normals_mask: ${adaptive_normals_mask}
+        max_depth: ${max_depth}
+        image_shape: ${agent.desired_image_shape}
+        learnable_contact_preprocess_params: ${learnable_contact_preprocess_params}
+        learning_rate: 0.0001
+        weight_decay: 0.0
+        contact_model_name: ${contact_model_name}
+        zero_centered: false
+    true_action_history: ${true_action_history}
+num_train_frames_bc: 50000
+num_train_frames_drq: 1100000
+stddev_schedule_drq: linear(1.0,0.1,100000)
+task_name: FrankaInsertion-v1
+num_train_frames_vinn: 25000
+num_train_frames_diffusion: 1000000
+num_train_epochs_bc: 5000
+num_train_epochs_diffusion: 15000
+validate_every_epochs_bc: 5
+validate_every_epochs_diffusion: 250
+validate_diffusion_on_action_loss_every_epochs: 250
+train_eval_diffusion_on_action_loss_every_epochs: 250
+check_topk_every_epochs: 5
+check_topk_every_epochs_diffusion: 250
+save_snapshot_every_epochs_diffusion: 1500
+x_limit:
+- 0.2
+- 0.7
+y_limit:
+- -0.4
+- 0.4
+z_limit:
+- -0.05
+- 0.55
+home_displacement:
+- 0.55
+- 0.0
+- 0.55
+- 180.0
+- 0.0
+- 0.0
+enable_gripper: true
+start_with_gripper_open: true
+offset_mask:
+- 1
+- 1
+- 1
+- 1
+- 1
+- 1
+path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy
+feature_type: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1