diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..e5395f2dc7f2151ae024add0a0d5112412f5a3ea 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,19 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +212506/wandb/run-20241229_212512-oco0rjll/run-oco0rjll.wandb filter=lfs diff=lfs merge=lfs -text +212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/2_eval_2_56287c330ba45a21ceae.mp4 filter=lfs diff=lfs merge=lfs -text +212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/3_eval_3_d9096d3d3b1128c5f3af.mp4 filter=lfs diff=lfs merge=lfs -text +212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/1_eval_1_df2b523b5c69376abbb6.mp4 filter=lfs diff=lfs merge=lfs -text +212506/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/0_eval_0_7f4ade8f7a09940de9a3.mp4 filter=lfs diff=lfs merge=lfs -text +212506/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/4_eval_4_277cb7c2f38e9f60cb6a.mp4 filter=lfs diff=lfs merge=lfs -text +212506/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +212506/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +212506/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +212506/episode_rosbags/episode_0_2024-12-29-21-25-40.bag filter=lfs diff=lfs merge=lfs -text +212506/episode_rosbags/episode_4_2024-12-29-21-28-37.bag filter=lfs diff=lfs merge=lfs -text +212506/episode_rosbags/episode_3_2024-12-29-21-27-48.bag filter=lfs diff=lfs merge=lfs -text +212506/episode_rosbags/episode_1_2024-12-29-21-26-21.bag filter=lfs diff=lfs merge=lfs -text +212506/episode_rosbags/episode_2_2024-12-29-21-27-02.bag filter=lfs diff=lfs merge=lfs -text diff --git a/212505/.hydra/config.yaml b/212505/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82dcfe3b36af5b4b12dbe2009190dadbc2e4b813 --- /dev/null +++ b/212505/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1000_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${now:%H%M%S} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/212505/.hydra/hydra.yaml b/212505/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7caab4e2932088a3cdd0664fde2f1f6a27f5deef --- /dev/null +++ b/212505/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212505 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/212505/.hydra/overrides.yaml b/212505/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/212505/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/212505/eval_policy.log b/212505/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..dfa546efb0ecb8bddb64ca2ad185a7a945af41b8 --- /dev/null +++ b/212505/eval_policy.log @@ -0,0 +1,15 @@ +[2024-12-29 21:25:05,560][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-29 21:25:05,563][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2024-12-29 21:25:08,109][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/212506/.hydra/config.yaml b/212506/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82dcfe3b36af5b4b12dbe2009190dadbc2e4b813 --- /dev/null +++ b/212506/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1000_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${now:%H%M%S} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/212506/.hydra/hydra.yaml b/212506/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e5b08314a47c633632c3a61a3033e04a21c5971 --- /dev/null +++ b/212506/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/212506/.hydra/overrides.yaml b/212506/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/212506/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/212506/episode_rosbags/aligned_depth_to_color_K.npy b/212506/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/212506/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/212506/episode_rosbags/cam_tf_world.npy b/212506/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/212506/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/212506/episode_rosbags/color_K.npy b/212506/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/212506/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/212506/episode_rosbags/depth_K.npy b/212506/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/212506/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/212506/episode_rosbags/episode_0_2024-12-29-21-25-40.bag b/212506/episode_rosbags/episode_0_2024-12-29-21-25-40.bag new file mode 100644 index 0000000000000000000000000000000000000000..979bfeffb26129515c1b5647077252c00d9246e8 --- /dev/null +++ b/212506/episode_rosbags/episode_0_2024-12-29-21-25-40.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba693f8802f73208accf84a97be7b2ffaa4045f9cd1bf585588d215337d57c6d +size 1412451872 diff --git a/212506/episode_rosbags/episode_1_2024-12-29-21-26-21.bag b/212506/episode_rosbags/episode_1_2024-12-29-21-26-21.bag new file mode 100644 index 0000000000000000000000000000000000000000..58ebca721163391c09c92764cab0cae7f763dbf1 --- /dev/null +++ b/212506/episode_rosbags/episode_1_2024-12-29-21-26-21.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fbf27e4be8f14b0e5f577f56b87cb2c3ccad19123dfb3d85f0644417a140aa6 +size 1409020913 diff --git a/212506/episode_rosbags/episode_2_2024-12-29-21-27-02.bag b/212506/episode_rosbags/episode_2_2024-12-29-21-27-02.bag new file mode 100644 index 0000000000000000000000000000000000000000..f23f8c8bed08279b8b85ca6dccc8819359b62190 --- /dev/null +++ b/212506/episode_rosbags/episode_2_2024-12-29-21-27-02.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bab15a1dbe3ae1bb6559090cbb454ca7b4ff3d45cfa42d16273899c874697569 +size 1412403096 diff --git a/212506/episode_rosbags/episode_3_2024-12-29-21-27-48.bag b/212506/episode_rosbags/episode_3_2024-12-29-21-27-48.bag new file mode 100644 index 0000000000000000000000000000000000000000..41d70e4853dcc60a8aacfed0ac7aefe92f140c68 --- /dev/null +++ b/212506/episode_rosbags/episode_3_2024-12-29-21-27-48.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bae008cd6450def998807252ef5909a28fb8565b1b6470274062047fef95a887 +size 1412697632 diff --git a/212506/episode_rosbags/episode_4_2024-12-29-21-28-37.bag b/212506/episode_rosbags/episode_4_2024-12-29-21-28-37.bag new file mode 100644 index 0000000000000000000000000000000000000000..ed50bef441047f287668ccd8ca5530c8c516d9a7 --- /dev/null +++ b/212506/episode_rosbags/episode_4_2024-12-29-21-28-37.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb556cd15568cfac79f3fb4223132ae8500d8f923e914953707279f998bef374 +size 1417097412 diff --git a/212506/eval_robot.log b/212506/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..98eac1315b441dc8740f70ae96fd50e7895955db --- /dev/null +++ b/212506/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-29 21:25:06,752][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-29 21:25:06,755][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/212506/eval_video/0_eval.mp4 b/212506/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..202fe5d7e4f337fb04e429108f72704a4ffd9036 --- /dev/null +++ b/212506/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f4ade8f7a09940de9a392152321c741514d0b2b83b953d2bc85b2587d34b370 +size 1160403 diff --git a/212506/eval_video/1_eval.mp4 b/212506/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b5818472294e79a15592a3db87ad607566636384 --- /dev/null +++ b/212506/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df2b523b5c69376abbb661917fe8d60d3ad79f771878de2226ab5c89077a3630 +size 1160649 diff --git a/212506/eval_video/2_eval.mp4 b/212506/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..16833afc43a80e2e928d70f8cceeed292671b60a --- /dev/null +++ b/212506/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56287c330ba45a21ceae2f5ada5b8060ed510846ec2d5a3b90730f83caff3226 +size 1157924 diff --git a/212506/eval_video/3_eval.mp4 b/212506/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a94d9ab2756a6a990b037a03a36683d601875642 --- /dev/null +++ b/212506/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9096d3d3b1128c5f3af34fcf97e720fde5df69b1403591cedf85594195eab69 +size 1175711 diff --git a/212506/eval_video/4_eval.mp4 b/212506/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..67f8d64209b5aa3249d69b15c75619a4127319b4 --- /dev/null +++ b/212506/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:277cb7c2f38e9f60cb6a1957cbae5c162329b3ce1d0991b1503aabd5e0830c0b +size 1163227 diff --git a/212506/tb/events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 b/212506/tb/events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 new file mode 100644 index 0000000000000000000000000000000000000000..53c0eba6a6c66bff4651cbb499edd251d306406e --- /dev/null +++ b/212506/tb/events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15b0fd95ea5588a3883740617d70ae416761eb477b73201865f1bf7a92e8e0c5 +size 1103 diff --git a/212506/wandb/debug-internal.log b/212506/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..471a85fc44a690cc015228dd46677c7cd6235565 --- /dev/null +++ b/212506/wandb/debug-internal.log @@ -0,0 +1,736 @@ +2024-12-29 21:25:12,629 INFO StreamThr :1360479 [internal.py:wandb_internal():85] W&B internal server running at pid: 1360479, started at: 2024-12-29 21:25:12.629472 +2024-12-29 21:25:12,631 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status +2024-12-29 21:25:12,641 INFO WriterThread:1360479 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/run-oco0rjll.wandb +2024-12-29 21:25:12,641 DEBUG SenderThread:1360479 [sender.py:send():379] send: header +2024-12-29 21:25:12,651 DEBUG SenderThread:1360479 [sender.py:send():379] send: run +2024-12-29 21:25:12,989 INFO SenderThread:1360479 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files +2024-12-29 21:25:12,989 INFO SenderThread:1360479 [sender.py:_start_run_threads():1188] run started: oco0rjll with start time 1735525512.630035 +2024-12-29 21:25:13,005 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: check_version +2024-12-29 21:25:13,006 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: check_version +2024-12-29 21:25:13,088 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: run_start +2024-12-29 21:25:13,093 DEBUG HandlerThread:1360479 [system_info.py:__init__():26] System info init +2024-12-29 21:25:13,093 DEBUG HandlerThread:1360479 [system_info.py:__init__():41] System info init done +2024-12-29 21:25:13,093 INFO HandlerThread:1360479 [system_monitor.py:start():194] Starting system monitor +2024-12-29 21:25:13,094 INFO SystemMonitor:1360479 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-29 21:25:13,094 INFO HandlerThread:1360479 [system_monitor.py:probe():214] Collecting system info +2024-12-29 21:25:13,094 INFO SystemMonitor:1360479 [interfaces.py:start():188] Started cpu monitoring +2024-12-29 21:25:13,095 INFO SystemMonitor:1360479 [interfaces.py:start():188] Started disk monitoring +2024-12-29 21:25:13,095 INFO SystemMonitor:1360479 [interfaces.py:start():188] Started gpu monitoring +2024-12-29 21:25:13,096 INFO SystemMonitor:1360479 [interfaces.py:start():188] Started memory monitoring +2024-12-29 21:25:13,096 INFO SystemMonitor:1360479 [interfaces.py:start():188] Started network monitoring +2024-12-29 21:25:13,109 DEBUG HandlerThread:1360479 [system_info.py:probe():152] Probing system +2024-12-29 21:25:13,110 DEBUG HandlerThread:1360479 [system_info.py:_probe_git():137] Probing git +2024-12-29 21:25:13,115 DEBUG HandlerThread:1360479 [system_info.py:_probe_git():145] Probing git done +2024-12-29 21:25:13,115 DEBUG HandlerThread:1360479 [system_info.py:probe():200] Probing system done +2024-12-29 21:25:13,115 DEBUG HandlerThread:1360479 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-30T02:25:13.109189', 'startedAt': '2024-12-30T02:25:12.622611', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '23581857e509febdae7f0ac45cc9cf4f9081f85c'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3762.1391249999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4520.366, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4546.057, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 477.0821304321289}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-29 21:25:13,115 INFO HandlerThread:1360479 [system_monitor.py:probe():224] Finished collecting system info +2024-12-29 21:25:13,115 INFO HandlerThread:1360479 [system_monitor.py:probe():227] Publishing system info +2024-12-29 21:25:13,115 DEBUG HandlerThread:1360479 [system_info.py:_save_code():44] Saving code +2024-12-29 21:25:13,116 DEBUG HandlerThread:1360479 [system_info.py:_save_code():67] Saving code done +2024-12-29 21:25:13,116 DEBUG HandlerThread:1360479 [system_info.py:_save_patches():84] Saving git patches +2024-12-29 21:25:13,179 DEBUG HandlerThread:1360479 [system_info.py:_save_patches():126] Saving git patches done +2024-12-29 21:25:13,179 INFO HandlerThread:1360479 [system_monitor.py:probe():229] Finished publishing system info +2024-12-29 21:25:13,182 DEBUG SenderThread:1360479 [sender.py:send():379] send: files +2024-12-29 21:25:13,182 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-29 21:25:13,183 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-29 21:25:13,183 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-29 21:25:13,418 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: python_packages +2024-12-29 21:25:13,418 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: python_packages +2024-12-29 21:25:13,419 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:25:13,419 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:25:13,420 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:13,422 INFO HandlerThread:1360479 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/tb" + save: true +} +control { + relay_id: "371900e530ae4f619f94a7297cc74e9c" +} +_info { + stream_id: "oco0rjll" +} + +2024-12-29 21:25:13,422 DEBUG HandlerThread:1360479 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-29 21:25:13,452 INFO wandb-upload_2:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/97vjqa7c-diff.patch +2024-12-29 21:25:13,476 INFO wandb-upload_1:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/1k2sknsw-code/FISH/eval_robot.py +2024-12-29 21:25:13,479 INFO wandb-upload_0:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/qurbt2cg-wandb-metadata.json +2024-12-29 21:25:13,521 DEBUG SenderThread:1360479 [sender.py:send():379] send: telemetry +2024-12-29 21:25:13,521 DEBUG SenderThread:1360479 [sender.py:send():379] send: tbrecord +2024-12-29 21:25:13,522 DEBUG SenderThread:1360479 [sender.py:send():379] send: files +2024-12-29 21:25:13,522 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 with policy live +2024-12-29 21:25:13,710 INFO wandb-upload_2:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/f1ksd97h-events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 +2024-12-29 21:25:13,991 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/requirements.txt +2024-12-29 21:25:13,991 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/code/FISH/eval_robot.py +2024-12-29 21:25:13,991 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:13,991 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/diff.patch +2024-12-29 21:25:13,991 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-metadata.json +2024-12-29 21:25:13,992 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 +2024-12-29 21:25:13,992 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/code +2024-12-29 21:25:13,992 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/code/FISH +2024-12-29 21:25:14,419 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:15,420 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:15,990 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:16,420 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:17,420 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:17,663 DEBUG SenderThread:1360479 [sender.py:send():379] send: config +2024-12-29 21:25:17,663 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:25:18,420 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:19,421 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:19,991 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:20,421 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:21,425 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:22,433 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:23,419 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:25:23,421 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:23,992 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:24,421 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:25,423 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:26,423 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:27,423 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:27,993 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:28,418 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:25:28,419 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:25:28,461 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:28,526 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:25:29,423 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:30,423 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:31,423 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:32,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:33,423 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:34,419 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:25:34,423 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:35,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:36,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:37,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:37,996 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:38,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:39,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:39,997 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:40,419 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:25:40,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:41,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:41,997 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:42,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:43,419 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:25:43,419 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:25:43,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:43,998 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:44,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:45,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:45,603 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:25:45,998 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:45,999 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/config.yaml +2024-12-29 21:25:46,425 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:47,425 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:47,999 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:48,425 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:49,425 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:49,999 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:50,425 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:50,804 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:25:51,425 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:52,000 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:52,425 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:53,425 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:54,001 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:54,426 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:55,426 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:55,997 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:25:56,001 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:56,426 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:57,426 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:58,002 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:58,419 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:25:58,419 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:25:58,426 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:59,427 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:00,002 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:00,427 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:01,003 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:01,428 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:02,004 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:02,428 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:03,428 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:04,004 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:04,428 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:05,428 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:06,005 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:06,194 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:06,429 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:07,429 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:08,005 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:08,429 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:08,601 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 21:26:08,602 DEBUG SenderThread:1360479 [sender.py:send():379] send: files +2024-12-29 21:26:08,602 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_7f4ade8f7a09940de9a3.mp4 with policy now +2024-12-29 21:26:08,602 DEBUG SenderThread:1360479 [sender.py:send():379] send: history +2024-12-29 21:26:08,602 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:26:08,603 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:26:08,857 INFO wandb-upload_0:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/r82fuotw-media/videos/eval/0_eval_0_7f4ade8f7a09940de9a3.mp4 +2024-12-29 21:26:09,006 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json +2024-12-29 21:26:09,006 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/0_eval_0_7f4ade8f7a09940de9a3.mp4 +2024-12-29 21:26:09,006 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos +2024-12-29 21:26:09,006 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval +2024-12-29 21:26:09,006 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media +2024-12-29 21:26:09,266 INFO wandb-upload_2:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/4nfn6llk-media/videos/eval/0_eval_0_7f4ade8f7a09940de9a3.mp4 +2024-12-29 21:26:09,429 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:10,006 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:10,430 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:11,430 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:11,603 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:12,430 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:13,097 DEBUG SystemMonitor:1360479 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-29 21:26:13,097 DEBUG SenderThread:1360479 [sender.py:send():379] send: stats +2024-12-29 21:26:13,420 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:26:13,421 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:26:13,461 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:14,007 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:14,430 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:15,430 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:16,008 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:16,431 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:16,934 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:17,431 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:18,009 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 +2024-12-29 21:26:18,259 INFO wandb-upload_0:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/znjalm43-events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 +2024-12-29 21:26:18,431 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:19,431 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:20,010 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:20,431 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:21,431 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:21,965 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:22,432 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:23,431 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:24,011 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:24,432 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:25,432 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:26,012 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:26,432 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:27,006 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:27,432 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:28,012 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:28,419 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:26:28,419 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:26:28,432 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:29,432 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:30,013 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:30,432 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:31,432 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:32,014 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:32,202 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:32,432 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:33,433 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:34,015 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:34,433 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:35,433 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:36,015 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:36,434 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:37,204 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:37,434 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:38,016 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:38,434 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:39,434 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:40,017 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:40,434 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:41,435 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:42,018 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:42,396 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:42,435 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:43,098 DEBUG SenderThread:1360479 [sender.py:send():379] send: stats +2024-12-29 21:26:43,419 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:26:43,420 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:26:43,435 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:44,018 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:44,435 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:45,435 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:46,019 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:46,435 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:47,397 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:47,435 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:48,019 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:48,435 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:49,437 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:49,687 DEBUG SenderThread:1360479 [sender.py:send():379] send: files +2024-12-29 21:26:49,687 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_df2b523b5c69376abbb6.mp4 with policy now +2024-12-29 21:26:49,688 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 21:26:49,688 DEBUG SenderThread:1360479 [sender.py:send():379] send: history +2024-12-29 21:26:49,689 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:26:49,689 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:26:49,933 INFO wandb-upload_2:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/epacwhln-media/videos/eval/1_eval_1_df2b523b5c69376abbb6.mp4 +2024-12-29 21:26:50,020 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json +2024-12-29 21:26:50,020 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:50,021 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/1_eval_1_df2b523b5c69376abbb6.mp4 +2024-12-29 21:26:50,021 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval +2024-12-29 21:26:50,292 INFO wandb-upload_0:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/uuduyhwz-media/videos/eval/1_eval_1_df2b523b5c69376abbb6.mp4 +2024-12-29 21:26:50,438 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:51,451 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:52,021 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:52,447 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:52,709 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:53,444 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:54,446 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:55,445 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:56,022 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:56,440 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:57,440 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:58,023 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:58,133 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:58,437 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:26:58,438 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:26:58,440 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:58,807 DEBUG SenderThread:1360479 [sender.py:send():379] send: history +2024-12-29 21:26:58,807 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:26:58,812 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:26:59,023 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json +2024-12-29 21:26:59,023 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 +2024-12-29 21:26:59,449 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:00,023 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:00,441 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:01,441 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:02,024 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:02,441 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:03,225 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:27:03,441 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:04,025 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:04,441 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:05,441 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:06,026 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:06,441 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:07,442 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:08,027 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:08,238 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:27:08,481 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:09,442 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:10,027 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:10,442 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:11,442 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:12,028 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:12,442 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:13,098 DEBUG SenderThread:1360479 [sender.py:send():379] send: stats +2024-12-29 21:27:13,420 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:27:13,420 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:27:13,442 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:13,532 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:27:14,029 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:14,442 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:15,443 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:16,030 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:16,443 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:17,443 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:18,030 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:18,485 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:18,642 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:27:19,443 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:20,031 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:20,443 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:21,443 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:22,032 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:22,443 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:23,481 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:23,837 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:27:24,032 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:24,444 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:25,444 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:26,033 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:26,444 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:27,444 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:28,034 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:28,420 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:27:28,421 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:27:28,444 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:29,035 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:27:29,456 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:30,035 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:30,357 DEBUG SenderThread:1360479 [sender.py:send():379] send: files +2024-12-29 21:27:30,358 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_56287c330ba45a21ceae.mp4 with policy now +2024-12-29 21:27:30,363 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 21:27:30,364 DEBUG SenderThread:1360479 [sender.py:send():379] send: history +2024-12-29 21:27:30,364 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:27:30,365 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:27:30,456 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:30,610 INFO wandb-upload_2:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/of9cesez-media/videos/eval/2_eval_2_56287c330ba45a21ceae.mp4 +2024-12-29 21:27:31,035 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json +2024-12-29 21:27:31,035 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/2_eval_2_56287c330ba45a21ceae.mp4 +2024-12-29 21:27:31,035 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval +2024-12-29 21:27:31,310 INFO wandb-upload_0:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/p7ulzqh4-media/videos/eval/2_eval_2_56287c330ba45a21ceae.mp4 +2024-12-29 21:27:31,457 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:32,035 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:32,452 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:33,452 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:34,073 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:27:34,454 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:35,455 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:36,037 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:36,466 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:37,457 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:38,037 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:38,453 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:39,445 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:27:39,453 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:40,456 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:41,457 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:42,465 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:43,099 DEBUG SenderThread:1360479 [sender.py:send():379] send: stats +2024-12-29 21:27:43,420 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:27:43,421 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:27:43,466 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:44,468 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:44,552 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:27:45,040 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 +2024-12-29 21:27:45,286 INFO wandb-upload_2:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/w7589ncm-events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 +2024-12-29 21:27:45,454 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:45,649 DEBUG SenderThread:1360479 [sender.py:send():379] send: history +2024-12-29 21:27:45,649 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:27:45,649 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:27:46,040 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json +2024-12-29 21:27:46,040 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:46,455 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:47,460 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:48,041 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:48,467 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:49,458 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:49,834 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:27:50,041 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:50,459 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:51,459 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:52,042 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:52,459 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:53,497 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:54,043 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:54,459 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:54,870 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:27:55,459 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:56,043 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:56,459 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:57,460 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:58,044 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:58,420 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:27:58,420 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:27:58,460 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:59,460 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:59,941 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:00,045 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:00,460 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:01,460 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:02,046 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:02,460 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:03,501 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:04,046 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:04,462 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:05,062 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:05,462 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:06,047 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:06,462 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:07,462 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:08,048 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:08,509 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:09,463 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:10,049 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:10,256 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:10,462 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:11,463 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:12,049 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:12,463 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:13,099 DEBUG SenderThread:1360479 [sender.py:send():379] send: stats +2024-12-29 21:28:13,420 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:28:13,420 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:28:13,463 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:14,050 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:14,463 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:15,263 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:15,463 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:16,051 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:16,472 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:17,233 DEBUG SenderThread:1360479 [sender.py:send():379] send: files +2024-12-29 21:28:17,234 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_d9096d3d3b1128c5f3af.mp4 with policy now +2024-12-29 21:28:17,238 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 21:28:17,239 DEBUG SenderThread:1360479 [sender.py:send():379] send: history +2024-12-29 21:28:17,239 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:28:17,239 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:28:17,466 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:17,502 INFO wandb-upload_0:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/09zme4ma-media/videos/eval/3_eval_3_d9096d3d3b1128c5f3af.mp4 +2024-12-29 21:28:18,051 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json +2024-12-29 21:28:18,052 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:18,052 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/3_eval_3_d9096d3d3b1128c5f3af.mp4 +2024-12-29 21:28:18,052 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval +2024-12-29 21:28:18,332 INFO wandb-upload_2:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/8rm19avh-media/videos/eval/3_eval_3_d9096d3d3b1128c5f3af.mp4 +2024-12-29 21:28:18,475 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:19,477 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:20,457 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:20,481 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:21,477 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:22,053 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:22,489 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:23,483 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:24,483 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:25,481 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:25,785 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:26,054 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:26,485 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:27,483 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:28,430 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:28:28,430 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:28:28,489 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:29,482 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:30,481 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:31,483 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:31,580 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:32,490 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:33,487 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:33,701 DEBUG SenderThread:1360479 [sender.py:send():379] send: history +2024-12-29 21:28:33,701 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:28:33,701 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:28:34,057 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json +2024-12-29 21:28:34,057 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:34,057 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 +2024-12-29 21:28:34,492 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:35,493 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:36,057 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:36,502 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:37,009 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:37,508 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:38,058 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:38,541 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:39,507 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:40,059 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:40,507 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:41,507 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:42,060 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:42,119 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:42,507 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:43,100 DEBUG SenderThread:1360479 [sender.py:send():379] send: stats +2024-12-29 21:28:43,427 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:28:43,427 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:28:43,507 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:44,060 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:44,507 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:45,508 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:46,061 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:46,508 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:47,125 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:47,508 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:48,062 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:48,508 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:49,508 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:50,063 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:50,508 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:51,508 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:52,064 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:52,130 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:52,509 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:53,508 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:54,064 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:54,509 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:55,509 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:56,065 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:56,509 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:57,322 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:57,509 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:58,066 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:58,427 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:28:58,428 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:28:58,509 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:59,509 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:00,067 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:29:00,509 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:01,509 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:02,067 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:29:02,509 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:02,524 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:29:03,510 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:04,068 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:29:04,510 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:05,517 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:05,589 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 21:29:05,589 DEBUG SenderThread:1360479 [sender.py:send():379] send: files +2024-12-29 21:29:05,589 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_277cb7c2f38e9f60cb6a.mp4 with policy now +2024-12-29 21:29:05,590 DEBUG SenderThread:1360479 [sender.py:send():379] send: history +2024-12-29 21:29:05,590 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:29:05,590 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:29:05,834 INFO wandb-upload_0:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/6o3pw2m5-media/videos/eval/4_eval_4_277cb7c2f38e9f60cb6a.mp4 +2024-12-29 21:29:06,069 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json +2024-12-29 21:29:06,069 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:29:06,069 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/4_eval_4_277cb7c2f38e9f60cb6a.mp4 +2024-12-29 21:29:06,069 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval +2024-12-29 21:29:06,324 INFO wandb-upload_2:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/njaqymsx-media/videos/eval/4_eval_4_277cb7c2f38e9f60cb6a.mp4 +2024-12-29 21:29:06,517 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:07,528 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:07,611 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:29:08,069 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:29:08,520 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:09,520 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:10,070 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:29:10,538 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:11,525 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:12,530 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:13,100 DEBUG SenderThread:1360479 [sender.py:send():379] send: stats +2024-12-29 21:29:13,101 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:29:13,443 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:29:13,444 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:29:13,549 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:14,072 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:29:14,539 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:14,706 DEBUG SenderThread:1360479 [sender.py:send():379] send: exit +2024-12-29 21:29:14,706 INFO SenderThread:1360479 [sender.py:send_exit():586] handling exit code: 0 +2024-12-29 21:29:14,706 INFO SenderThread:1360479 [sender.py:send_exit():588] handling runtime: 241 +2024-12-29 21:29:14,721 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:29:14,721 INFO SenderThread:1360479 [sender.py:send_exit():594] send defer +2024-12-29 21:29:14,721 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:14,721 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-29 21:29:14,722 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:14,722 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-29 21:29:14,722 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 1 +2024-12-29 21:29:14,722 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:14,722 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-29 21:29:14,722 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:14,722 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-29 21:29:14,722 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 2 +2024-12-29 21:29:14,722 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:14,722 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-29 21:29:14,722 INFO HandlerThread:1360479 [system_monitor.py:finish():203] Stopping system monitor +2024-12-29 21:29:14,722 DEBUG SystemMonitor:1360479 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-29 21:29:14,722 INFO HandlerThread:1360479 [interfaces.py:finish():200] Joined cpu monitor +2024-12-29 21:29:14,722 DEBUG SystemMonitor:1360479 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-29 21:29:14,723 INFO HandlerThread:1360479 [interfaces.py:finish():200] Joined disk monitor +2024-12-29 21:29:14,773 INFO HandlerThread:1360479 [interfaces.py:finish():200] Joined gpu monitor +2024-12-29 21:29:14,774 INFO HandlerThread:1360479 [interfaces.py:finish():200] Joined memory monitor +2024-12-29 21:29:14,774 INFO HandlerThread:1360479 [interfaces.py:finish():200] Joined network monitor +2024-12-29 21:29:14,774 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:14,774 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-29 21:29:14,774 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 3 +2024-12-29 21:29:14,775 DEBUG SenderThread:1360479 [sender.py:send():379] send: stats +2024-12-29 21:29:14,775 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:14,775 DEBUG SenderThread:1360479 [sender.py:send():379] send: history +2024-12-29 21:29:14,775 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-29 21:29:14,775 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:29:14,796 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:29:14,797 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:14,797 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-29 21:29:14,797 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 4 +2024-12-29 21:29:14,797 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:14,797 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-29 21:29:15,072 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json +2024-12-29 21:29:15,072 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 +2024-12-29 21:29:15,347 INFO wandb-upload_0:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/mr7i954z-events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 +2024-12-29 21:29:16,072 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:29:21,759 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:29:21,759 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:29:21,759 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:21,759 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: keepalive +2024-12-29 21:29:21,759 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-29 21:29:21,759 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 5 +2024-12-29 21:29:21,759 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:29:21,760 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:21,760 DEBUG SenderThread:1360479 [sender.py:send():379] send: history +2024-12-29 21:29:21,760 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-29 21:29:21,760 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:29:21,760 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:29:21,760 DEBUG SenderThread:1360479 [sender.py:send():379] send: summary +2024-12-29 21:29:21,761 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:29:21,761 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:21,761 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-29 21:29:21,761 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 6 +2024-12-29 21:29:21,761 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:21,761 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-29 21:29:21,761 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:21,761 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-29 21:29:21,761 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 7 +2024-12-29 21:29:21,761 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:29:21,761 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:21,762 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-29 21:29:21,762 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:21,762 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-29 21:29:22,074 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json +2024-12-29 21:29:22,707 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:29:23,721 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 8 +2024-12-29 21:29:23,721 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:29:23,721 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:23,721 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-29 21:29:23,722 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:23,722 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-29 21:29:23,722 INFO SenderThread:1360479 [job_builder.py:build():440] Attempting to build job artifact +2024-12-29 21:29:23,724 INFO SenderThread:1360479 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-29 21:29:23,738 INFO SenderThread:1360479 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-29 21:29:23,752 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 9 +2024-12-29 21:29:23,753 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:23,753 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-29 21:29:23,753 DEBUG SenderThread:1360479 [sender.py:send():379] send: artifact +2024-12-29 21:29:24,075 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:29:24,192 INFO wandb-upload_1:1360479 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpu0f2y331 +2024-12-29 21:29:24,192 INFO wandb-upload_0:1360479 [upload_job.py:push():85] Skipped uploading /tmp/tmp6dzmspxx/wandb-job.json +2024-12-29 21:29:24,336 INFO wandb-upload_2:1360479 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpf1m3m3qp +2024-12-29 21:29:24,708 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:29:24,896 INFO SenderThread:1360479 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxMjA3MTYzNw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxMjA2MzIwOA==', 'versionIndex': 387}}} +2024-12-29 21:29:24,896 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:24,896 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-29 21:29:24,896 INFO SenderThread:1360479 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-29 21:29:25,076 INFO SenderThread:1360479 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files +2024-12-29 21:29:25,076 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/config.yaml config.yaml +2024-12-29 21:29:25,076 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json wandb-summary.json +2024-12-29 21:29:25,077 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/requirements.txt requirements.txt +2024-12-29 21:29:25,078 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-metadata.json wandb-metadata.json +2024-12-29 21:29:25,078 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 +2024-12-29 21:29:25,079 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log output.log +2024-12-29 21:29:25,079 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/diff.patch diff.patch +2024-12-29 21:29:25,083 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-29 21:29:25,084 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/1_eval_1_df2b523b5c69376abbb6.mp4 media/videos/eval/1_eval_1_df2b523b5c69376abbb6.mp4 +2024-12-29 21:29:25,084 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/2_eval_2_56287c330ba45a21ceae.mp4 media/videos/eval/2_eval_2_56287c330ba45a21ceae.mp4 +2024-12-29 21:29:25,084 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/4_eval_4_277cb7c2f38e9f60cb6a.mp4 media/videos/eval/4_eval_4_277cb7c2f38e9f60cb6a.mp4 +2024-12-29 21:29:25,084 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/3_eval_3_d9096d3d3b1128c5f3af.mp4 media/videos/eval/3_eval_3_d9096d3d3b1128c5f3af.mp4 +2024-12-29 21:29:25,084 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/0_eval_0_7f4ade8f7a09940de9a3.mp4 media/videos/eval/0_eval_0_7f4ade8f7a09940de9a3.mp4 +2024-12-29 21:29:25,084 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 10 +2024-12-29 21:29:25,084 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:29:25,084 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:25,084 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-29 21:29:25,085 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:25,085 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-29 21:29:25,085 INFO SenderThread:1360479 [file_pusher.py:finish():169] shutting down file pusher +2024-12-29 21:29:25,270 INFO wandb-upload_2:1360479 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json +2024-12-29 21:29:25,292 INFO wandb-upload_3:1360479 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:29:25,297 INFO wandb-upload_1:1360479 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/config.yaml +2024-12-29 21:29:25,317 INFO wandb-upload_0:1360479 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/requirements.txt +2024-12-29 21:29:25,517 INFO Thread-11 (_thread_body):1360479 [sender.py:transition_state():614] send defer: 11 +2024-12-29 21:29:25,517 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:25,517 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-29 21:29:25,517 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:25,517 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-29 21:29:25,517 INFO SenderThread:1360479 [file_pusher.py:join():175] waiting for file pusher +2024-12-29 21:29:25,518 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 12 +2024-12-29 21:29:25,518 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:25,518 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-29 21:29:25,518 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:25,519 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-29 21:29:25,519 INFO SenderThread:1360479 [file_stream.py:finish():601] file stream finish called +2024-12-29 21:29:25,708 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:29:25,725 INFO SenderThread:1360479 [file_stream.py:finish():605] file stream finish is done +2024-12-29 21:29:25,725 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 13 +2024-12-29 21:29:25,725 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:29:25,725 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:25,725 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-29 21:29:25,725 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:25,725 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-29 21:29:25,725 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 14 +2024-12-29 21:29:25,726 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:25,726 DEBUG SenderThread:1360479 [sender.py:send():379] send: final +2024-12-29 21:29:25,726 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-29 21:29:25,726 DEBUG SenderThread:1360479 [sender.py:send():379] send: footer +2024-12-29 21:29:25,726 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:25,726 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-29 21:29:25,726 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:29:25,726 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:29:25,727 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: server_info +2024-12-29 21:29:25,727 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: server_info +2024-12-29 21:29:25,728 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:29:25,728 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: get_summary +2024-12-29 21:29:25,729 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-29 21:29:25,729 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:25,782 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:29:25,783 INFO MainThread:1360479 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-29 21:29:25,783 INFO MainThread:1360479 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-29 21:29:25,783 INFO MainThread:1360479 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-29 21:29:25,783 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: shutdown +2024-12-29 21:29:25,783 INFO HandlerThread:1360479 [handler.py:finish():882] shutting down handler +2024-12-29 21:29:26,728 INFO WriterThread:1360479 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/run-oco0rjll.wandb +2024-12-29 21:29:26,783 INFO SenderThread:1360479 [sender.py:finish():1615] shutting down sender +2024-12-29 21:29:26,783 INFO SenderThread:1360479 [file_pusher.py:finish():169] shutting down file pusher +2024-12-29 21:29:26,783 INFO SenderThread:1360479 [file_pusher.py:join():175] waiting for file pusher diff --git a/212506/wandb/debug.log b/212506/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f58fcb13bfaf67c74135f2592ae4895b10943301 --- /dev/null +++ b/212506/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-29 21:25:12,623 INFO MainThread:1360290 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_setup.py:_flush():76] Configure stats pid to 1360290 +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/settings +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/logs/debug.log +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/logs/debug-internal.log +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_init.py:init():569] calling init triggers +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1000_0', 'wandb_notes': '1000_0_req_1351_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506'} +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_init.py:init():619] starting backend +2024-12-29 21:25:12,625 INFO MainThread:1360290 [wandb_init.py:init():623] setting up manager +2024-12-29 21:25:12,628 INFO MainThread:1360290 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-29 21:25:12,629 INFO MainThread:1360290 [wandb_init.py:init():631] backend started and connected +2024-12-29 21:25:12,640 INFO MainThread:1360290 [wandb_init.py:init():720] updated telemetry +2024-12-29 21:25:12,649 INFO MainThread:1360290 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-29 21:25:13,005 INFO MainThread:1360290 [wandb_run.py:_on_init():2435] communicating current version +2024-12-29 21:25:13,082 INFO MainThread:1360290 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-29 21:25:13,082 INFO MainThread:1360290 [wandb_init.py:init():804] starting run threads in backend +2024-12-29 21:25:13,419 INFO MainThread:1360290 [wandb_run.py:_console_start():2413] atexit reg +2024-12-29 21:25:13,419 INFO MainThread:1360290 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-29 21:25:13,419 INFO MainThread:1360290 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-29 21:25:13,420 INFO MainThread:1360290 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-29 21:25:13,421 INFO MainThread:1360290 [wandb_init.py:init():847] run started, returning control to user process +2024-12-29 21:25:13,422 INFO MainThread:1360290 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/tb, True +2024-12-29 21:25:17,662 INFO MainThread:1360290 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2024-12-29 21:29:26,788 WARNING MsgRouterThr:1360290 [router.py:message_loop():77] message_loop has been closed diff --git a/212506/wandb/run-20241229_212512-oco0rjll/files/code/FISH/eval_robot.py b/212506/wandb/run-20241229_212512-oco0rjll/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..75a95b02db44bf1d6cdce4329109f52a0c4c1bc8 --- /dev/null +++ b/212506/wandb/run-20241229_212512-oco0rjll/files/code/FISH/eval_robot.py @@ -0,0 +1,512 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/212506/wandb/run-20241229_212512-oco0rjll/files/config.yaml b/212506/wandb/run-20241229_212512-oco0rjll/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2db0cac94b11af0d93032c9f53bb80d66005010f --- /dev/null +++ b/212506/wandb/run-20241229_212512-oco0rjll/files/config.yaml @@ -0,0 +1,893 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1000_0' +wandb_notes: + desc: null + value: 1000_0_req_1351_0restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 0 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735525512 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 16 + - 23 + - 35 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/212506/wandb/run-20241229_212512-oco0rjll/files/diff.patch b/212506/wandb/run-20241229_212512-oco0rjll/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..f8dd03a05ea08752d22903a02e415dfdcc04630f --- /dev/null +++ b/212506/wandb/run-20241229_212512-oco0rjll/files/diff.patch @@ -0,0 +1,196 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 2303343..566e9c7 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -71,7 +71,7 @@ dtc_adaptive_normalization: false + mask_normals_within_sdf: true + adaptive_normals_mask: true + learnable_contact_preprocess_params: false +-contact_model_name: 'local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9' ++contact_model_name: 'local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9' + # contact_model_name: 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9' + contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt' # no mask no flow, just context + # contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/197406_2/checkpoints/epoch=08-val_loss=0.00.ckpt' # w/ mask no flow, just context +@@ -123,6 +123,14 @@ bc_weight_type: 'qfilter' # linear, qfilter + + # Load weights + load_checkpoint: ${agent.load_checkpoint} ++# 120 demos ++# RGBD+act history ++# wandb_run_id: '1045_0' # greece ++# wandb_run_id: '1051_0' # all ++ ++# RGBD+mask+act history ++wandb_run_id: '1000_0' # all ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +@@ -144,7 +152,7 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '523_0' # fowlers + # wandb_run_id: '524_0' # lib + # wandb_run_id: '525_0' # modelsys +-wandb_run_id: '526_0' # electrodyn ++# wandb_run_id: '526_0' # electrodyn + + # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 32 held out demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 0f225ec..5ed3eb4 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -15,8 +15,8 @@ if not local_checkpoint_root_dir.exists(): + local_checkpoint_root_dir.mkdir(parents=True) + + # Download the model checkpoints +-# remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1') +-remote_checkpoint_root_dir = Path('/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1') ++remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1') ++# remote_checkpoint_root_dir = Path('/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1') + remote_checkpoint_root_dir = remote_checkpoint_root_dir.expanduser() + remote_username = 'leonmkim' + remote_host = 'grasp-login1' +@@ -146,6 +146,11 @@ remote_host = 'grasp-login1' + # '265627', + # '265620', + # ] ++ ++run_id_list = [ ++ '1000_0', ++] ++ + checkpoint_epoch = 12000 + + # use subprocess to download the model checkpoints in parallel +Submodule ros_ws/src/contact_estimation_ros contains modified content +diff --git a/ros_ws/src/contact_estimation_ros/fish_data_collection/src/rosbag_handler.py b/ros_ws/src/contact_estimation_ros/fish_data_collection/src/rosbag_handler.py +index dcbcfa4..35dd7e8 100755 +--- a/ros_ws/src/contact_estimation_ros/fish_data_collection/src/rosbag_handler.py ++++ b/ros_ws/src/contact_estimation_ros/fish_data_collection/src/rosbag_handler.py +@@ -15,6 +15,8 @@ import time + import datetime + import shutil + ++from scipy.spatial.transform import Rotation as R ++ + # solution to launching from main thread from: https://answers.ros.org/question/260212/start-launchfile-from-service/ + # from Queue import Queue + +@@ -23,7 +25,7 @@ from fish_data_collection.srv import StartBagging, StartBaggingRequest, StartBag + from sensor_msgs.msg import CameraInfo + + class RosbaggingHandler(): +- def __init__(self, base_dset_dir, experiment_dirname, bag_launch_path, demo_dirname, camera_calibration_dir, ++ def __init__(self, base_dset_dir, experiment_dirname, bag_launch_path, demo_dirname, + multicam=False, align_depth_to_color=False): + ## bag files and target files should be identified by episode id + self.align_depth_to_color = align_depth_to_color +@@ -33,8 +35,6 @@ class RosbaggingHandler(): + self.intrinsics_saved = False + self.extrinsics_saved = False + +- self.camera_calibration_dir = camera_calibration_dir +- + self.full_bag_dir = os.path.join(self.base_dset_dir, self.experiment_dirname, self.demo_dirname) + os.makedirs(self.full_bag_dir, exist_ok=True) + +@@ -63,7 +63,7 @@ class RosbaggingHandler(): + self.abort_bagging_request = False + + self.save_camera_intrinsics() +- self.save_camera_extrinsics(self.camera_calibration_dir) ++ self.save_camera_extrinsics() + + while not rospy.is_shutdown(): + if self.start_bagging_request: +@@ -83,23 +83,56 @@ class RosbaggingHandler(): + self.start_bagging_request = True + return response + +- def save_camera_extrinsics(self, path_to_calibration_dir): ++ def save_camera_extrinsics(self): + if not self.extrinsics_saved: +- # copy over the contents of the calibration directory, but not the directory itself, into the bag directory +- # list the contents of the calibration directory that end in .npy +- calibration_dir_contents = glob.glob(os.path.join(path_to_calibration_dir, '*.npy')) +- # calibration_dir_contents = os.listdir(path_to_calibration_dir) +- # copy the contents of the calibration directory into the bag directory +- for item in calibration_dir_contents: +- # make sure the item is a file +- if os.path.isfile(item): +- # print('copying {} to {}'.format(item, os.path.join(self.full_bag_dir, os.path.basename(item)))) +- shutil.copy(item, os.path.join(self.full_bag_dir, os.path.basename(item))) +- # also save the path to the calibration directory as a txt file +- with open(os.path.join(self.full_bag_dir, 'calibration_dir.txt'), 'w') as f: +- f.write(path_to_calibration_dir) ++ # get cam extrinsic ++ # use ros tf2 to get transform from color_optical_frame to panda_link0 ++ import tf2_ros ++ tfBuffer = tf2_ros.Buffer() ++ listener = tf2_ros.TransformListener(tfBuffer) ++ ++ # get transform from color_optical_frame to panda_link0 ++ # target, source where target is the parent frame and source is the child frame ++ color_tf_world_ros = tfBuffer.lookup_transform('camera_color_optical_frame', 'panda_link0', rospy.Time(0), rospy.Duration(1.0)).transform ++ self.color_tf_world = np.eye(4) ++ self.color_tf_world[0, 3] =color_tf_world_ros.translation.x ++ self.color_tf_world[1, 3] =color_tf_world_ros.translation.y ++ self.color_tf_world[2, 3] =color_tf_world_ros.translation.z ++ self.color_tf_world[:3, :3] = R.from_quat([color_tf_world_ros.rotation.x, color_tf_world_ros.rotation.y, color_tf_world_ros.rotation.z, color_tf_world_ros.rotation.w]).as_matrix() ++ ++ # dump the info to the bag directory ++ np.save(os.path.join(self.full_bag_dir, 'color_tf_world.npy'), self.color_tf_world) ++ ++ depth_tf_world_ros = tfBuffer.lookup_transform('camera_depth_optical_frame', 'panda_link0', rospy.Time(0), rospy.Duration(1.0)).transform ++ self.depth_tf_world = np.eye(4) ++ self.depth_tf_world[0, 3] = depth_tf_world_ros.translation.x ++ self.depth_tf_world[1, 3] = depth_tf_world_ros.translation.y ++ self.depth_tf_world[2, 3] = depth_tf_world_ros.translation.z ++ self.depth_tf_world[:3, :3] = \ ++ R.from_quat([ ++ depth_tf_world_ros.rotation.x, ++ depth_tf_world_ros.rotation.y, ++ depth_tf_world_ros.rotation.z, ++ depth_tf_world_ros.rotation.w, ++ ]).as_matrix() ++ ++ np.save(os.path.join(self.full_bag_dir, 'depth_tf_world.npy'), self.depth_tf_world) ++ ++ # # copy over the contents of the calibration directory, but not the directory itself, into the bag directory ++ # # list the contents of the calibration directory that end in .npy ++ # calibration_dir_contents = glob.glob(os.path.join(path_to_calibration_dir, '*.npy')) ++ # # calibration_dir_contents = os.listdir(path_to_calibration_dir) ++ # # copy the contents of the calibration directory into the bag directory ++ # for item in calibration_dir_contents: ++ # # make sure the item is a file ++ # if os.path.isfile(item): ++ # # print('copying {} to {}'.format(item, os.path.join(self.full_bag_dir, os.path.basename(item)))) ++ # shutil.copy(item, os.path.join(self.full_bag_dir, os.path.basename(item))) ++ # # also save the path to the calibration directory as a txt file ++ # with open(os.path.join(self.full_bag_dir, 'calibration_dir.txt'), 'w') as f: ++ # f.write(path_to_calibration_dir) + +- self.extrinsics_saved = True ++ # self.extrinsics_saved = True + + def save_camera_intrinsics(self): + if not self.intrinsics_saved: +@@ -198,9 +231,6 @@ class RosbaggingHandler(): + if self.bag_recording: + self.current_recording_bag_path = bag_path + # response.success = True +- +- +- + # return response + + def stop_bagging_service_callback(self, req): +@@ -304,7 +334,7 @@ if __name__ == '__main__': + assert os.path.exists(rosbag_launch_path), "rosbag launch file does not exist at path: {}".format(rosbag_launch_path) + + camera_calibration_dir = '/home/leonmkim/FISH_franka_ros_ws/src/contact_estimation_ros/fish_data_collection/scripts/camera_poses_L515/20240313-153328' +- dc_handler = RosbaggingHandler(base_dset_dir, experiment_dirname, rosbag_launch_path, demo_dirname, camera_calibration_dir, multicam=multicam, align_depth_to_color=align_depth_to_color) ++ dc_handler = RosbaggingHandler(base_dset_dir, experiment_dirname, rosbag_launch_path, demo_dirname, multicam=multicam, align_depth_to_color=align_depth_to_color) + + # rospy.spin() + diff --git a/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/0_eval_0_7f4ade8f7a09940de9a3.mp4 b/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/0_eval_0_7f4ade8f7a09940de9a3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..202fe5d7e4f337fb04e429108f72704a4ffd9036 --- /dev/null +++ b/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/0_eval_0_7f4ade8f7a09940de9a3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f4ade8f7a09940de9a392152321c741514d0b2b83b953d2bc85b2587d34b370 +size 1160403 diff --git a/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/1_eval_1_df2b523b5c69376abbb6.mp4 b/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/1_eval_1_df2b523b5c69376abbb6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b5818472294e79a15592a3db87ad607566636384 --- /dev/null +++ b/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/1_eval_1_df2b523b5c69376abbb6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df2b523b5c69376abbb661917fe8d60d3ad79f771878de2226ab5c89077a3630 +size 1160649 diff --git a/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/2_eval_2_56287c330ba45a21ceae.mp4 b/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/2_eval_2_56287c330ba45a21ceae.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..16833afc43a80e2e928d70f8cceeed292671b60a --- /dev/null +++ b/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/2_eval_2_56287c330ba45a21ceae.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56287c330ba45a21ceae2f5ada5b8060ed510846ec2d5a3b90730f83caff3226 +size 1157924 diff --git a/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/3_eval_3_d9096d3d3b1128c5f3af.mp4 b/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/3_eval_3_d9096d3d3b1128c5f3af.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a94d9ab2756a6a990b037a03a36683d601875642 --- /dev/null +++ b/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/3_eval_3_d9096d3d3b1128c5f3af.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9096d3d3b1128c5f3af34fcf97e720fde5df69b1403591cedf85594195eab69 +size 1175711 diff --git a/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/4_eval_4_277cb7c2f38e9f60cb6a.mp4 b/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/4_eval_4_277cb7c2f38e9f60cb6a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..67f8d64209b5aa3249d69b15c75619a4127319b4 --- /dev/null +++ b/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/4_eval_4_277cb7c2f38e9f60cb6a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:277cb7c2f38e9f60cb6a1957cbae5c162329b3ce1d0991b1503aabd5e0830c0b +size 1163227 diff --git a/212506/wandb/run-20241229_212512-oco0rjll/files/output.log b/212506/wandb/run-20241229_212512-oco0rjll/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..3a61257a3e180cb1789a236550b95243585fb99e --- /dev/null +++ b/212506/wandb/run-20241229_212512-oco0rjll/files/output.log @@ -0,0 +1,6202 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735525517.663253]: resetting environment +[INFO] [1735525517.666453]: cleared current plan +[INFO] [1735525517.667945]: moving to home +[INFO] [1735525518.670009]: reached home +[INFO] [1735525518.671432]: reset action history +[INFO] [1735525521.382336]: environment reset +Starting episode 0 +[INFO] [1735525521.384854]: resetting environment +[INFO] [1735525521.387626]: cleared current plan +[INFO] [1735525521.388383]: moving to home +[INFO] [1735525522.390650]: reached home +[INFO] [1735525522.393897]: reset action history +[INFO] [1735525525.102905]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735525536.926355]: resetting environment +[INFO] [1735525536.929078]: cleared current plan +[INFO] [1735525536.929853]: moving to home +[INFO] [1735525537.932259]: reached home +[INFO] [1735525537.933954]: reset action history +[INFO] [1735525540.641230]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/episode_rosbags/episode_0_2024-12-29-21-25-40.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1360290.log +started roslaunch server http://158.130.50.37:42517/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1360581] +process[print_text-2]: started with pid [1360582] +process[rosbag_record-3]: started with pid [1360605] +started bagging! +[INFO] [1735525541.406330]: received mask_obs with header seq: 1 +stamp: + secs: 1735525541 + nsecs: 401127099 +frame_id: '' and grasped_object_mask header seq: 64183 +stamp: + secs: 1735525541 + nsecs: 161967754 +frame_id: "camera_color_optical_frame" +[INFO] [1735525541.595862]: received mask_obs with header seq: 2 +stamp: + secs: 1735525541 + nsecs: 596911191 +frame_id: '' and grasped_object_mask header seq: 64190 +stamp: + secs: 1735525541 + nsecs: 395109177 +frame_id: "camera_color_optical_frame" +[INFO] [1735525542.004822]: received mask_obs with header seq: 3 +stamp: + secs: 1735525542 + nsecs: 6027221 +frame_id: '' and grasped_object_mask header seq: 64202 +stamp: + secs: 1735525541 + nsecs: 794790268 +frame_id: "camera_color_optical_frame" +[INFO] [1735525542.189023]: received mask_obs with header seq: 4 +stamp: + secs: 1735525542 + nsecs: 190337419 +frame_id: '' and grasped_object_mask header seq: 64207 +stamp: + secs: 1735525541 + nsecs: 961324453 +frame_id: "camera_color_optical_frame" +[INFO] [1735525542.368650]: received mask_obs with header seq: 5 +stamp: + secs: 1735525542 + nsecs: 369979619 +frame_id: '' and grasped_object_mask header seq: 64213 +stamp: + secs: 1735525542 + nsecs: 161165237 +frame_id: "camera_color_optical_frame" +[INFO] [1735525542.537896]: received mask_obs with header seq: 6 +stamp: + secs: 1735525542 + nsecs: 538835048 +frame_id: '' and grasped_object_mask header seq: 64218 +stamp: + secs: 1735525542 + nsecs: 327698231 +frame_id: "camera_color_optical_frame" +[INFO] [1735525542.719201]: received mask_obs with header seq: 7 +stamp: + secs: 1735525542 + nsecs: 720126390 +frame_id: '' and grasped_object_mask header seq: 64223 +stamp: + secs: 1735525542 + nsecs: 494228601 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 106773972 for nearest: 1735525542767531752 - target: 1735525542660757780 at idx 28 +[INFO] [1735525542.896971]: received mask_obs with header seq: 8 +stamp: + secs: 1735525542 + nsecs: 898154497 +frame_id: '' and grasped_object_mask header seq: 64228 +stamp: + secs: 1735525542 + nsecs: 660757780 +frame_id: "camera_color_optical_frame" +[INFO] [1735525543.109743]: received mask_obs with header seq: 9 +stamp: + secs: 1735525543 + nsecs: 110584974 +frame_id: '' and grasped_object_mask header seq: 64234 +stamp: + secs: 1735525542 + nsecs: 860593557 +frame_id: "camera_color_optical_frame" +[INFO] [1735525543.307935]: received mask_obs with header seq: 10 +stamp: + secs: 1735525543 + nsecs: 308567523 +frame_id: '' and grasped_object_mask header seq: 64241 +stamp: + secs: 1735525543 + nsecs: 93735218 +frame_id: "camera_color_optical_frame" +[INFO] [1735525543.500115]: received mask_obs with header seq: 11 +stamp: + secs: 1735525543 + nsecs: 501284837 +frame_id: '' and grasped_object_mask header seq: 64246 +stamp: + secs: 1735525543 + nsecs: 260264874 +frame_id: "camera_color_optical_frame" +[INFO] [1735525543.692902]: received mask_obs with header seq: 12 +stamp: + secs: 1735525543 + nsecs: 693838596 +frame_id: '' and grasped_object_mask header seq: 64252 +stamp: + secs: 1735525543 + nsecs: 460103512 +frame_id: "camera_color_optical_frame" +[INFO] [1735525543.889773]: received mask_obs with header seq: 13 +stamp: + secs: 1735525543 + nsecs: 891079187 +frame_id: '' and grasped_object_mask header seq: 64258 +stamp: + secs: 1735525543 + nsecs: 659942389 +frame_id: "camera_color_optical_frame" +[INFO] [1735525544.072946]: received mask_obs with header seq: 14 +stamp: + secs: 1735525544 + nsecs: 74256896 +frame_id: '' and grasped_object_mask header seq: 64264 +stamp: + secs: 1735525543 + nsecs: 859781265 +frame_id: "camera_color_optical_frame" +[INFO] [1735525544.259632]: received mask_obs with header seq: 15 +stamp: + secs: 1735525544 + nsecs: 260287761 +frame_id: '' and grasped_object_mask header seq: 64269 +stamp: + secs: 1735525544 + nsecs: 26314259 +frame_id: "camera_color_optical_frame" +[INFO] [1735525544.440324]: received mask_obs with header seq: 16 +stamp: + secs: 1735525544 + nsecs: 441003799 +frame_id: '' and grasped_object_mask header seq: 64275 +stamp: + secs: 1735525544 + nsecs: 226153135 +frame_id: "camera_color_optical_frame" +[INFO] [1735525544.638055]: received mask_obs with header seq: 17 +stamp: + secs: 1735525544 + nsecs: 639369487 +frame_id: '' and grasped_object_mask header seq: 64281 +stamp: + secs: 1735525544 + nsecs: 425990820 +frame_id: "camera_color_optical_frame" +[INFO] [1735525544.832309]: received mask_obs with header seq: 18 +stamp: + secs: 1735525544 + nsecs: 833580017 +frame_id: '' and grasped_object_mask header seq: 64286 +stamp: + secs: 1735525544 + nsecs: 592522144 +frame_id: "camera_color_optical_frame" +[INFO] [1735525545.023990]: received mask_obs with header seq: 19 +stamp: + secs: 1735525545 + nsecs: 24982213 +frame_id: '' and grasped_object_mask header seq: 64292 +stamp: + secs: 1735525544 + nsecs: 792359352 +frame_id: "camera_color_optical_frame" +[INFO] [1735525545.208905]: received mask_obs with header seq: 20 +stamp: + secs: 1735525545 + nsecs: 208873748 +frame_id: '' and grasped_object_mask header seq: 64298 +stamp: + secs: 1735525544 + nsecs: 992196560 +frame_id: "camera_color_optical_frame" +[INFO] [1735525545.409056]: received mask_obs with header seq: 21 +stamp: + secs: 1735525545 + nsecs: 409623622 +frame_id: '' and grasped_object_mask header seq: 64304 +stamp: + secs: 1735525545 + nsecs: 192033768 +frame_id: "camera_color_optical_frame" +[INFO] [1735525545.599743]: received mask_obs with header seq: 22 +stamp: + secs: 1735525545 + nsecs: 600995302 +frame_id: '' and grasped_object_mask header seq: 64309 +stamp: + secs: 1735525545 + nsecs: 358982563 +frame_id: "camera_color_optical_frame" +[INFO] [1735525545.790383]: received mask_obs with header seq: 23 +stamp: + secs: 1735525545 + nsecs: 791622400 +frame_id: '' and grasped_object_mask header seq: 64315 +stamp: + secs: 1735525545 + nsecs: 561132669 +frame_id: "camera_color_optical_frame" +[INFO] [1735525545.991905]: received mask_obs with header seq: 24 +stamp: + secs: 1735525545 + nsecs: 993308067 +frame_id: '' and grasped_object_mask header seq: 64321 +stamp: + secs: 1735525545 + nsecs: 763369560 +frame_id: "camera_color_optical_frame" +[INFO] [1735525546.190035]: received mask_obs with header seq: 25 +stamp: + secs: 1735525546 + nsecs: 191378831 +frame_id: '' and grasped_object_mask header seq: 64327 +stamp: + secs: 1735525545 + nsecs: 965693712 +frame_id: "camera_color_optical_frame" +[INFO] [1735525546.391051]: received mask_obs with header seq: 26 +stamp: + secs: 1735525546 + nsecs: 392091989 +frame_id: '' and grasped_object_mask header seq: 64333 +stamp: + secs: 1735525546 + nsecs: 168104649 +frame_id: "camera_color_optical_frame" +[INFO] [1735525546.592495]: received mask_obs with header seq: 27 +stamp: + secs: 1735525546 + nsecs: 593689680 +frame_id: '' and grasped_object_mask header seq: 64339 +stamp: + secs: 1735525546 + nsecs: 370074987 +frame_id: "camera_color_optical_frame" +[INFO] [1735525546.793719]: received mask_obs with header seq: 28 +stamp: + secs: 1735525546 + nsecs: 795081377 +frame_id: '' and grasped_object_mask header seq: 64345 +stamp: + secs: 1735525546 + nsecs: 569693565 +frame_id: "camera_color_optical_frame" +[INFO] [1735525546.996789]: received mask_obs with header seq: 29 +stamp: + secs: 1735525546 + nsecs: 996847867 +frame_id: '' and grasped_object_mask header seq: 64351 +stamp: + secs: 1735525546 + nsecs: 769300938 +frame_id: "camera_color_optical_frame" +[INFO] [1735525547.195633]: received mask_obs with header seq: 30 +stamp: + secs: 1735525547 + nsecs: 196759223 +frame_id: '' and grasped_object_mask header seq: 64357 +stamp: + secs: 1735525546 + nsecs: 968896866 +frame_id: "camera_color_optical_frame" +[INFO] [1735525547.394005]: received mask_obs with header seq: 31 +stamp: + secs: 1735525547 + nsecs: 395340442 +frame_id: '' and grasped_object_mask header seq: 64363 +stamp: + secs: 1735525547 + nsecs: 168481350 +frame_id: "camera_color_optical_frame" +[INFO] [1735525547.597531]: received mask_obs with header seq: 32 +stamp: + secs: 1735525547 + nsecs: 598845005 +frame_id: '' and grasped_object_mask header seq: 64369 +stamp: + secs: 1735525547 + nsecs: 368062496 +frame_id: "camera_color_optical_frame" +[INFO] [1735525547.794244]: received mask_obs with header seq: 33 +stamp: + secs: 1735525547 + nsecs: 795571804 +frame_id: '' and grasped_object_mask header seq: 64375 +stamp: + secs: 1735525547 + nsecs: 567672968 +frame_id: "camera_color_optical_frame" +[INFO] [1735525547.998680]: received mask_obs with header seq: 34 +stamp: + secs: 1735525547 + nsecs: 999884843 +frame_id: '' and grasped_object_mask header seq: 64381 +stamp: + secs: 1735525547 + nsecs: 767270088 +frame_id: "camera_color_optical_frame" +[INFO] [1735525548.194787]: received mask_obs with header seq: 35 +stamp: + secs: 1735525548 + nsecs: 196068286 +frame_id: '' and grasped_object_mask header seq: 64387 +stamp: + secs: 1735525547 + nsecs: 966855764 +frame_id: "camera_color_optical_frame" +[INFO] [1735525548.394497]: received mask_obs with header seq: 36 +stamp: + secs: 1735525548 + nsecs: 395790100 +frame_id: '' and grasped_object_mask header seq: 64393 +stamp: + secs: 1735525548 + nsecs: 166429758 +frame_id: "camera_color_optical_frame" +[INFO] [1735525548.596096]: received mask_obs with header seq: 37 +stamp: + secs: 1735525548 + nsecs: 597311973 +frame_id: '' and grasped_object_mask header seq: 64399 +stamp: + secs: 1735525548 + nsecs: 366002083 +frame_id: "camera_color_optical_frame" +[INFO] [1735525548.794199]: received mask_obs with header seq: 38 +stamp: + secs: 1735525548 + nsecs: 795565843 +frame_id: '' and grasped_object_mask header seq: 64406 +stamp: + secs: 1735525548 + nsecs: 598883629 +frame_id: "camera_color_optical_frame" +[INFO] [1735525548.997822]: received mask_obs with header seq: 39 +stamp: + secs: 1735525548 + nsecs: 998982667 +frame_id: '' and grasped_object_mask header seq: 64411 +stamp: + secs: 1735525548 + nsecs: 765217781 +frame_id: "camera_color_optical_frame" +[INFO] [1735525549.194443]: received mask_obs with header seq: 40 +stamp: + secs: 1735525549 + nsecs: 195757865 +frame_id: '' and grasped_object_mask header seq: 64417 +stamp: + secs: 1735525548 + nsecs: 964807749 +frame_id: "camera_color_optical_frame" +[INFO] [1735525549.393826]: received mask_obs with header seq: 41 +stamp: + secs: 1735525549 + nsecs: 395148038 +frame_id: '' and grasped_object_mask header seq: 64424 +stamp: + secs: 1735525549 + nsecs: 197647572 +frame_id: "camera_color_optical_frame" +[INFO] [1735525549.596735]: received mask_obs with header seq: 42 +stamp: + secs: 1735525549 + nsecs: 598065614 +frame_id: '' and grasped_object_mask header seq: 64430 +stamp: + secs: 1735525549 + nsecs: 397233248 +frame_id: "camera_color_optical_frame" +[INFO] [1735525549.795732]: received mask_obs with header seq: 43 +stamp: + secs: 1735525549 + nsecs: 797078609 +frame_id: '' and grasped_object_mask header seq: 64436 +stamp: + secs: 1735525549 + nsecs: 596845627 +frame_id: "camera_color_optical_frame" +[INFO] [1735525549.998733]: received mask_obs with header seq: 44 +stamp: + secs: 1735525550 + nsecs: 76770 +frame_id: '' and grasped_object_mask header seq: 64442 +stamp: + secs: 1735525549 + nsecs: 796446800 +frame_id: "camera_color_optical_frame" +[INFO] [1735525550.204628]: received mask_obs with header seq: 45 +stamp: + secs: 1735525550 + nsecs: 205931425 +frame_id: '' and grasped_object_mask header seq: 64448 +stamp: + secs: 1735525549 + nsecs: 996036291 +frame_id: "camera_color_optical_frame" +[INFO] [1735525550.399477]: received mask_obs with header seq: 46 +stamp: + secs: 1735525550 + nsecs: 400667428 +frame_id: '' and grasped_object_mask header seq: 64454 +stamp: + secs: 1735525550 + nsecs: 195614815 +frame_id: "camera_color_optical_frame" +[INFO] [1735525550.607008]: received mask_obs with header seq: 47 +stamp: + secs: 1735525550 + nsecs: 607890129 +frame_id: '' and grasped_object_mask header seq: 64460 +stamp: + secs: 1735525550 + nsecs: 395200491 +frame_id: "camera_color_optical_frame" +[INFO] [1735525550.803361]: received mask_obs with header seq: 48 +stamp: + secs: 1735525550 + nsecs: 803928852 +frame_id: '' and grasped_object_mask header seq: 64466 +stamp: + secs: 1735525550 + nsecs: 594814539 +frame_id: "camera_color_optical_frame" +[INFO] [1735525551.004580]: received mask_obs with header seq: 49 +stamp: + secs: 1735525551 + nsecs: 5728960 +frame_id: '' and grasped_object_mask header seq: 64472 +stamp: + secs: 1735525550 + nsecs: 794415712 +frame_id: "camera_color_optical_frame" +[INFO] [1735525551.199701]: received mask_obs with header seq: 50 +stamp: + secs: 1735525551 + nsecs: 200806140 +frame_id: '' and grasped_object_mask header seq: 64478 +stamp: + secs: 1735525550 + nsecs: 994005680 +frame_id: "camera_color_optical_frame" +[INFO] [1735525551.398664]: received mask_obs with header seq: 51 +stamp: + secs: 1735525551 + nsecs: 399650573 +frame_id: '' and grasped_object_mask header seq: 64484 +stamp: + secs: 1735525551 + nsecs: 193583965 +frame_id: "camera_color_optical_frame" +[INFO] [1735525551.600424]: received mask_obs with header seq: 52 +stamp: + secs: 1735525551 + nsecs: 601765632 +frame_id: '' and grasped_object_mask header seq: 64490 +stamp: + secs: 1735525551 + nsecs: 393170595 +frame_id: "camera_color_optical_frame" +[INFO] [1735525551.798589]: received mask_obs with header seq: 53 +stamp: + secs: 1735525551 + nsecs: 799802541 +frame_id: '' and grasped_object_mask header seq: 64496 +stamp: + secs: 1735525551 + nsecs: 592785120 +frame_id: "camera_color_optical_frame" +[INFO] [1735525551.998571]: received mask_obs with header seq: 54 +stamp: + secs: 1735525551 + nsecs: 999876499 +frame_id: '' and grasped_object_mask header seq: 64502 +stamp: + secs: 1735525551 + nsecs: 792389154 +frame_id: "camera_color_optical_frame" +[INFO] [1735525552.199071]: received mask_obs with header seq: 55 +stamp: + secs: 1735525552 + nsecs: 200246334 +frame_id: '' and grasped_object_mask header seq: 64508 +stamp: + secs: 1735525551 + nsecs: 991980791 +frame_id: "camera_color_optical_frame" +[INFO] [1735525552.399420]: received mask_obs with header seq: 56 +stamp: + secs: 1735525552 + nsecs: 400692701 +frame_id: '' and grasped_object_mask header seq: 64514 +stamp: + secs: 1735525552 + nsecs: 191560745 +frame_id: "camera_color_optical_frame" +[INFO] [1735525552.602646]: received mask_obs with header seq: 57 +stamp: + secs: 1735525552 + nsecs: 602798700 +frame_id: '' and grasped_object_mask header seq: 64520 +stamp: + secs: 1735525552 + nsecs: 391144991 +frame_id: "camera_color_optical_frame" +[INFO] [1735525552.804544]: received mask_obs with header seq: 58 +stamp: + secs: 1735525552 + nsecs: 805716037 +frame_id: '' and grasped_object_mask header seq: 64526 +stamp: + secs: 1735525552 + nsecs: 590748787 +frame_id: "camera_color_optical_frame" +[INFO] [1735525553.005657]: received mask_obs with header seq: 59 +stamp: + secs: 1735525553 + nsecs: 6870269 +frame_id: '' and grasped_object_mask header seq: 64532 +stamp: + secs: 1735525552 + nsecs: 790340900 +frame_id: "camera_color_optical_frame" +[INFO] [1735525553.208918]: received mask_obs with header seq: 60 +stamp: + secs: 1735525553 + nsecs: 210209369 +frame_id: '' and grasped_object_mask header seq: 64538 +stamp: + secs: 1735525552 + nsecs: 989921808 +frame_id: "camera_color_optical_frame" +[INFO] [1735525553.404607]: received mask_obs with header seq: 61 +stamp: + secs: 1735525553 + nsecs: 405981779 +frame_id: '' and grasped_object_mask header seq: 64544 +stamp: + secs: 1735525553 + nsecs: 189490080 +frame_id: "camera_color_optical_frame" +[INFO] [1735525553.603977]: received mask_obs with header seq: 62 +stamp: + secs: 1735525553 + nsecs: 604930400 +frame_id: '' and grasped_object_mask header seq: 64550 +stamp: + secs: 1735525553 + nsecs: 389068127 +frame_id: "camera_color_optical_frame" +[INFO] [1735525553.798643]: received mask_obs with header seq: 63 +stamp: + secs: 1735525553 + nsecs: 799988269 +frame_id: '' and grasped_object_mask header seq: 64556 +stamp: + secs: 1735525553 + nsecs: 588677883 +frame_id: "camera_color_optical_frame" +[INFO] [1735525553.997403]: received mask_obs with header seq: 64 +stamp: + secs: 1735525553 + nsecs: 998771905 +frame_id: '' and grasped_object_mask header seq: 64562 +stamp: + secs: 1735525553 + nsecs: 788276196 +frame_id: "camera_color_optical_frame" +[INFO] [1735525554.198032]: received mask_obs with header seq: 65 +stamp: + secs: 1735525554 + nsecs: 199205636 +frame_id: '' and grasped_object_mask header seq: 64568 +stamp: + secs: 1735525553 + nsecs: 987862825 +frame_id: "camera_color_optical_frame" +[INFO] [1735525554.396393]: received mask_obs with header seq: 66 +stamp: + secs: 1735525554 + nsecs: 397623062 +frame_id: '' and grasped_object_mask header seq: 64574 +stamp: + secs: 1735525554 + nsecs: 187437773 +frame_id: "camera_color_optical_frame" +[INFO] [1735525554.601481]: received mask_obs with header seq: 67 +stamp: + secs: 1735525554 + nsecs: 599877834 +frame_id: '' and grasped_object_mask header seq: 64580 +stamp: + secs: 1735525554 + nsecs: 387022734 +frame_id: "camera_color_optical_frame" +[INFO] [1735525554.797514]: received mask_obs with header seq: 68 +stamp: + secs: 1735525554 + nsecs: 798526763 +frame_id: '' and grasped_object_mask header seq: 64586 +stamp: + secs: 1735525554 + nsecs: 586636305 +frame_id: "camera_color_optical_frame" +[INFO] [1735525554.997774]: received mask_obs with header seq: 69 +stamp: + secs: 1735525554 + nsecs: 998989582 +frame_id: '' and grasped_object_mask header seq: 64592 +stamp: + secs: 1735525554 + nsecs: 786238432 +frame_id: "camera_color_optical_frame" +[INFO] [1735525555.197984]: received mask_obs with header seq: 70 +stamp: + secs: 1735525555 + nsecs: 198747158 +frame_id: '' and grasped_object_mask header seq: 64598 +stamp: + secs: 1735525554 + nsecs: 985829353 +frame_id: "camera_color_optical_frame" +[INFO] [1735525555.396592]: received mask_obs with header seq: 71 +stamp: + secs: 1735525555 + nsecs: 397713899 +frame_id: '' and grasped_object_mask header seq: 64604 +stamp: + secs: 1735525555 + nsecs: 185408592 +frame_id: "camera_color_optical_frame" +[INFO] [1735525555.599163]: received mask_obs with header seq: 72 +stamp: + secs: 1735525555 + nsecs: 598852157 +frame_id: '' and grasped_object_mask header seq: 64610 +stamp: + secs: 1735525555 + nsecs: 384995222 +frame_id: "camera_color_optical_frame" +[INFO] [1735525555.796797]: received mask_obs with header seq: 73 +stamp: + secs: 1735525555 + nsecs: 797980308 +frame_id: '' and grasped_object_mask header seq: 64616 +stamp: + secs: 1735525555 + nsecs: 584608555 +frame_id: "camera_color_optical_frame" +[INFO] [1735525555.996469]: received mask_obs with header seq: 74 +stamp: + secs: 1735525555 + nsecs: 997821331 +frame_id: '' and grasped_object_mask header seq: 64622 +stamp: + secs: 1735525555 + nsecs: 784209490 +frame_id: "camera_color_optical_frame" +[INFO] [1735525556.196700]: received mask_obs with header seq: 75 +stamp: + secs: 1735525556 + nsecs: 197813987 +frame_id: '' and grasped_object_mask header seq: 64628 +stamp: + secs: 1735525555 + nsecs: 983798504 +frame_id: "camera_color_optical_frame" +[INFO] [1735525556.396029]: received mask_obs with header seq: 76 +stamp: + secs: 1735525556 + nsecs: 397361755 +frame_id: '' and grasped_object_mask header seq: 64634 +stamp: + secs: 1735525556 + nsecs: 183376312 +frame_id: "camera_color_optical_frame" +[INFO] [1735525556.601314]: received mask_obs with header seq: 77 +stamp: + secs: 1735525556 + nsecs: 602484464 +frame_id: '' and grasped_object_mask header seq: 64640 +stamp: + secs: 1735525556 + nsecs: 382962942 +frame_id: "camera_color_optical_frame" +[INFO] [1735525556.798313]: received mask_obs with header seq: 78 +stamp: + secs: 1735525556 + nsecs: 797805547 +frame_id: '' and grasped_object_mask header seq: 64646 +stamp: + secs: 1735525556 + nsecs: 582578421 +frame_id: "camera_color_optical_frame" +[INFO] [1735525556.997745]: received mask_obs with header seq: 79 +stamp: + secs: 1735525556 + nsecs: 997498035 +frame_id: '' and grasped_object_mask header seq: 64652 +stamp: + secs: 1735525556 + nsecs: 782183409 +frame_id: "camera_color_optical_frame" +[INFO] [1735525557.196928]: received mask_obs with header seq: 80 +stamp: + secs: 1735525557 + nsecs: 196767330 +frame_id: '' and grasped_object_mask header seq: 64658 +stamp: + secs: 1735525556 + nsecs: 981775761 +frame_id: "camera_color_optical_frame" +[INFO] [1735525557.397704]: received mask_obs with header seq: 81 +stamp: + secs: 1735525557 + nsecs: 398851633 +frame_id: '' and grasped_object_mask header seq: 64664 +stamp: + secs: 1735525557 + nsecs: 181356668 +frame_id: "camera_color_optical_frame" +[INFO] [1735525557.596615]: received mask_obs with header seq: 82 +stamp: + secs: 1735525557 + nsecs: 597738265 +frame_id: '' and grasped_object_mask header seq: 64670 +stamp: + secs: 1735525557 + nsecs: 380944967 +frame_id: "camera_color_optical_frame" +[INFO] [1735525557.797223]: received mask_obs with header seq: 83 +stamp: + secs: 1735525557 + nsecs: 796346902 +frame_id: '' and grasped_object_mask header seq: 64676 +stamp: + secs: 1735525557 + nsecs: 580559969 +frame_id: "camera_color_optical_frame" +[INFO] [1735525557.997786]: received mask_obs with header seq: 84 +stamp: + secs: 1735525557 + nsecs: 996863842 +frame_id: '' and grasped_object_mask header seq: 64682 +stamp: + secs: 1735525557 + nsecs: 780163527 +frame_id: "camera_color_optical_frame" +[INFO] [1735525558.198409]: received mask_obs with header seq: 85 +stamp: + secs: 1735525558 + nsecs: 197691202 +frame_id: '' and grasped_object_mask header seq: 64688 +stamp: + secs: 1735525557 + nsecs: 979756594 +frame_id: "camera_color_optical_frame" +[INFO] [1735525558.397793]: received mask_obs with header seq: 86 +stamp: + secs: 1735525558 + nsecs: 398721456 +frame_id: '' and grasped_object_mask header seq: 64694 +stamp: + secs: 1735525558 + nsecs: 179337025 +frame_id: "camera_color_optical_frame" +[INFO] [1735525558.599549]: received mask_obs with header seq: 87 +stamp: + secs: 1735525558 + nsecs: 600761175 +frame_id: '' and grasped_object_mask header seq: 64700 +stamp: + secs: 1735525558 + nsecs: 378923655 +frame_id: "camera_color_optical_frame" +[INFO] [1735525558.795269]: received mask_obs with header seq: 88 +stamp: + secs: 1735525558 + nsecs: 796245574 +frame_id: '' and grasped_object_mask header seq: 64706 +stamp: + secs: 1735525558 + nsecs: 578533888 +frame_id: "camera_color_optical_frame" +[INFO] [1735525558.996605]: received mask_obs with header seq: 89 +stamp: + secs: 1735525558 + nsecs: 996017694 +frame_id: '' and grasped_object_mask header seq: 64712 +stamp: + secs: 1735525558 + nsecs: 778132677 +frame_id: "camera_color_optical_frame" +[INFO] [1735525559.196532]: received mask_obs with header seq: 90 +stamp: + secs: 1735525559 + nsecs: 197222709 +frame_id: '' and grasped_object_mask header seq: 64718 +stamp: + secs: 1735525558 + nsecs: 977719307 +frame_id: "camera_color_optical_frame" +[INFO] [1735525559.397171]: received mask_obs with header seq: 91 +stamp: + secs: 1735525559 + nsecs: 398563623 +frame_id: '' and grasped_object_mask header seq: 64724 +stamp: + secs: 1735525559 + nsecs: 177295446 +frame_id: "camera_color_optical_frame" +[INFO] [1735525559.596795]: received mask_obs with header seq: 92 +stamp: + secs: 1735525559 + nsecs: 598008632 +frame_id: '' and grasped_object_mask header seq: 64730 +stamp: + secs: 1735525559 + nsecs: 376877546 +frame_id: "camera_color_optical_frame" +[INFO] [1735525559.796891]: received mask_obs with header seq: 93 +stamp: + secs: 1735525559 + nsecs: 797701120 +frame_id: '' and grasped_object_mask header seq: 64736 +stamp: + secs: 1735525559 + nsecs: 576484442 +frame_id: "camera_color_optical_frame" +[INFO] [1735525559.997281]: received mask_obs with header seq: 94 +stamp: + secs: 1735525559 + nsecs: 998347282 +frame_id: '' and grasped_object_mask header seq: 64742 +stamp: + secs: 1735525559 + nsecs: 776079416 +frame_id: "camera_color_optical_frame" +[INFO] [1735525560.196497]: received mask_obs with header seq: 95 +stamp: + secs: 1735525560 + nsecs: 196178674 +frame_id: '' and grasped_object_mask header seq: 64748 +stamp: + secs: 1735525559 + nsecs: 975662708 +frame_id: "camera_color_optical_frame" +[INFO] [1735525560.396528]: received mask_obs with header seq: 96 +stamp: + secs: 1735525560 + nsecs: 397010326 +frame_id: '' and grasped_object_mask header seq: 64754 +stamp: + secs: 1735525560 + nsecs: 175233841 +frame_id: "camera_color_optical_frame" +[INFO] [1735525560.598824]: received mask_obs with header seq: 97 +stamp: + secs: 1735525560 + nsecs: 599962949 +frame_id: '' and grasped_object_mask header seq: 64760 +stamp: + secs: 1735525560 + nsecs: 374813557 +frame_id: "camera_color_optical_frame" +[INFO] [1735525560.796707]: received mask_obs with header seq: 98 +stamp: + secs: 1735525560 + nsecs: 797211885 +frame_id: '' and grasped_object_mask header seq: 64766 +stamp: + secs: 1735525560 + nsecs: 574419022 +frame_id: "camera_color_optical_frame" +[INFO] [1735525561.002686]: received mask_obs with header seq: 99 +stamp: + secs: 1735525561 + nsecs: 3723382 +frame_id: '' and grasped_object_mask header seq: 64772 +stamp: + secs: 1735525560 + nsecs: 774012089 +frame_id: "camera_color_optical_frame" +[INFO] [1735525561.201918]: received mask_obs with header seq: 100 +stamp: + secs: 1735525561 + nsecs: 203274488 +frame_id: '' and grasped_object_mask header seq: 64778 +stamp: + secs: 1735525560 + nsecs: 973593473 +frame_id: "camera_color_optical_frame" +[INFO] [1735525561.395787]: received mask_obs with header seq: 101 +stamp: + secs: 1735525561 + nsecs: 397083759 +frame_id: '' and grasped_object_mask header seq: 64784 +stamp: + secs: 1735525561 + nsecs: 173162937 +frame_id: "camera_color_optical_frame" +[INFO] [1735525561.597498]: received mask_obs with header seq: 102 +stamp: + secs: 1735525561 + nsecs: 598847389 +frame_id: '' and grasped_object_mask header seq: 64790 +stamp: + secs: 1735525561 + nsecs: 373156071 +frame_id: "camera_color_optical_frame" +[INFO] [1735525561.794855]: received mask_obs with header seq: 103 +stamp: + secs: 1735525561 + nsecs: 795445919 +frame_id: '' and grasped_object_mask header seq: 64796 +stamp: + secs: 1735525561 + nsecs: 574074507 +frame_id: "camera_color_optical_frame" +[INFO] [1735525561.995029]: received mask_obs with header seq: 104 +stamp: + secs: 1735525561 + nsecs: 995963573 +frame_id: '' and grasped_object_mask header seq: 64802 +stamp: + secs: 1735525561 + nsecs: 775081158 +frame_id: "camera_color_optical_frame" +[INFO] [1735525562.195987]: received mask_obs with header seq: 105 +stamp: + secs: 1735525562 + nsecs: 195792913 +frame_id: '' and grasped_object_mask header seq: 64808 +stamp: + secs: 1735525561 + nsecs: 976173878 +frame_id: "camera_color_optical_frame" +[INFO] [1735525562.395430]: received mask_obs with header seq: 106 +stamp: + secs: 1735525562 + nsecs: 395537376 +frame_id: '' and grasped_object_mask header seq: 64814 +stamp: + secs: 1735525562 + nsecs: 177353621 +frame_id: "camera_color_optical_frame" +[INFO] [1735525562.599014]: received mask_obs with header seq: 107 +stamp: + secs: 1735525562 + nsecs: 600213050 +frame_id: '' and grasped_object_mask header seq: 64820 +stamp: + secs: 1735525562 + nsecs: 378168106 +frame_id: "camera_color_optical_frame" +[INFO] [1735525562.794859]: received mask_obs with header seq: 108 +stamp: + secs: 1735525562 + nsecs: 795958995 +frame_id: '' and grasped_object_mask header seq: 64826 +stamp: + secs: 1735525562 + nsecs: 578002691 +frame_id: "camera_color_optical_frame" +[INFO] [1735525562.995825]: received mask_obs with header seq: 109 +stamp: + secs: 1735525562 + nsecs: 996035337 +frame_id: '' and grasped_object_mask header seq: 64832 +stamp: + secs: 1735525562 + nsecs: 777836800 +frame_id: "camera_color_optical_frame" +[INFO] [1735525563.195675]: received mask_obs with header seq: 110 +stamp: + secs: 1735525563 + nsecs: 196287393 +frame_id: '' and grasped_object_mask header seq: 64838 +stamp: + secs: 1735525562 + nsecs: 977671862 +frame_id: "camera_color_optical_frame" +[INFO] [1735525563.394987]: received mask_obs with header seq: 111 +stamp: + secs: 1735525563 + nsecs: 395453214 +frame_id: '' and grasped_object_mask header seq: 64844 +stamp: + secs: 1735525563 + nsecs: 177505732 +frame_id: "camera_color_optical_frame" +[INFO] [1735525563.603803]: received mask_obs with header seq: 112 +stamp: + secs: 1735525563 + nsecs: 604072332 +frame_id: '' and grasped_object_mask header seq: 64850 +stamp: + secs: 1735525563 + nsecs: 377339363 +frame_id: "camera_color_optical_frame" +[INFO] [1735525563.794715]: received mask_obs with header seq: 113 +stamp: + secs: 1735525563 + nsecs: 795265197 +frame_id: '' and grasped_object_mask header seq: 64856 +stamp: + secs: 1735525563 + nsecs: 577172756 +frame_id: "camera_color_optical_frame" +[INFO] [1735525563.994942]: received mask_obs with header seq: 114 +stamp: + secs: 1735525563 + nsecs: 996218204 +frame_id: '' and grasped_object_mask header seq: 64862 +stamp: + secs: 1735525563 + nsecs: 777006149 +frame_id: "camera_color_optical_frame" +[INFO] [1735525564.194976]: received mask_obs with header seq: 115 +stamp: + secs: 1735525564 + nsecs: 196298599 +frame_id: '' and grasped_object_mask header seq: 64868 +stamp: + secs: 1735525563 + nsecs: 976839304 +frame_id: "camera_color_optical_frame" +[INFO] [1735525564.394676]: received mask_obs with header seq: 116 +stamp: + secs: 1735525564 + nsecs: 394973754 +frame_id: '' and grasped_object_mask header seq: 64874 +stamp: + secs: 1735525564 + nsecs: 176673412 +frame_id: "camera_color_optical_frame" +[INFO] [1735525564.599865]: received mask_obs with header seq: 117 +stamp: + secs: 1735525564 + nsecs: 601218461 +frame_id: '' and grasped_object_mask header seq: 64880 +stamp: + secs: 1735525564 + nsecs: 376508474 +frame_id: "camera_color_optical_frame" +[INFO] [1735525564.797881]: received mask_obs with header seq: 118 +stamp: + secs: 1735525564 + nsecs: 799114942 +frame_id: '' and grasped_object_mask header seq: 64886 +stamp: + secs: 1735525564 + nsecs: 576350212 +frame_id: "camera_color_optical_frame" +[INFO] [1735525564.993776]: received mask_obs with header seq: 119 +stamp: + secs: 1735525564 + nsecs: 994941473 +frame_id: '' and grasped_object_mask header seq: 64892 +stamp: + secs: 1735525564 + nsecs: 776191950 +frame_id: "camera_color_optical_frame" +[INFO] [1735525565.194287]: received mask_obs with header seq: 120 +stamp: + secs: 1735525565 + nsecs: 194786787 +frame_id: '' and grasped_object_mask header seq: 64898 +stamp: + secs: 1735525564 + nsecs: 976033926 +frame_id: "camera_color_optical_frame" +[INFO] [1735525565.394249]: received mask_obs with header seq: 121 +stamp: + secs: 1735525565 + nsecs: 394171953 +frame_id: '' and grasped_object_mask header seq: 64904 +stamp: + secs: 1735525565 + nsecs: 175875902 +frame_id: "camera_color_optical_frame" +[INFO] [1735525565.594311]: received mask_obs with header seq: 122 +stamp: + secs: 1735525565 + nsecs: 595666885 +frame_id: '' and grasped_object_mask header seq: 64910 +stamp: + secs: 1735525565 + nsecs: 375719547 +frame_id: "camera_color_optical_frame" +[INFO] [1735525565.793183]: received mask_obs with header seq: 123 +stamp: + secs: 1735525565 + nsecs: 793788909 +frame_id: '' and grasped_object_mask header seq: 64916 +stamp: + secs: 1735525565 + nsecs: 575562954 +frame_id: "camera_color_optical_frame" +[INFO] [1735525565.994156]: received mask_obs with header seq: 124 +stamp: + secs: 1735525565 + nsecs: 995142221 +frame_id: '' and grasped_object_mask header seq: 64922 +stamp: + secs: 1735525565 + nsecs: 775406599 +frame_id: "camera_color_optical_frame" +[INFO] [1735525566.193463]: received mask_obs with header seq: 125 +stamp: + secs: 1735525566 + nsecs: 194451093 +frame_id: '' and grasped_object_mask header seq: 64928 +stamp: + secs: 1735525565 + nsecs: 975250483 +frame_id: "camera_color_optical_frame" +[INFO] [1735525566.393579]: received mask_obs with header seq: 126 +stamp: + secs: 1735525566 + nsecs: 394596576 +frame_id: '' and grasped_object_mask header seq: 64934 +stamp: + secs: 1735525566 + nsecs: 175094366 +frame_id: "camera_color_optical_frame" +[INFO] [1735525566.596479]: received mask_obs with header seq: 127 +stamp: + secs: 1735525566 + nsecs: 597842454 +frame_id: '' and grasped_object_mask header seq: 64940 +stamp: + secs: 1735525566 + nsecs: 374936819 +frame_id: "camera_color_optical_frame" +[INFO] [1735525566.803090]: received mask_obs with header seq: 128 +stamp: + secs: 1735525566 + nsecs: 804438114 +frame_id: '' and grasped_object_mask header seq: 64946 +stamp: + secs: 1735525566 + nsecs: 574775457 +frame_id: "camera_color_optical_frame" +[INFO] [1735525566.996264]: received mask_obs with header seq: 129 +stamp: + secs: 1735525566 + nsecs: 997464895 +frame_id: '' and grasped_object_mask header seq: 64952 +stamp: + secs: 1735525566 + nsecs: 774613142 +frame_id: "camera_color_optical_frame" +[INFO] [1735525567.193328]: received mask_obs with header seq: 130 +stamp: + secs: 1735525567 + nsecs: 194233894 +frame_id: '' and grasped_object_mask header seq: 64958 +stamp: + secs: 1735525566 + nsecs: 974450588 +frame_id: "camera_color_optical_frame" +[INFO] [1735525567.393365]: received mask_obs with header seq: 131 +stamp: + secs: 1735525567 + nsecs: 394733667 +frame_id: '' and grasped_object_mask header seq: 64964 +stamp: + secs: 1735525567 + nsecs: 174288273 +frame_id: "camera_color_optical_frame" +[INFO] [1735525567.596617]: received mask_obs with header seq: 132 +stamp: + secs: 1735525567 + nsecs: 598015546 +frame_id: '' and grasped_object_mask header seq: 64970 +stamp: + secs: 1735525567 + nsecs: 374127388 +frame_id: "camera_color_optical_frame" +[INFO] [1735525567.646579]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1735525567.803315]: received mask_obs with header seq: 133 +stamp: + secs: 1735525567 + nsecs: 804207563 +frame_id: '' and grasped_object_mask header seq: 64976 +stamp: + secs: 1735525567 + nsecs: 573970556 +frame_id: "camera_color_optical_frame" +[INFO] [1735525568.373310]: Stored episode 1. +[INFO] [1735525568.373542]: Saving video... +[INFO] [1735525568.560561]: Video saved! +Starting episode 1 +[INFO] [1735525568.575939]: resetting environment +[INFO] [1735525568.581963]: cleared current plan +[INFO] [1735525568.582179]: moving to home +[INFO] [1735525572.184504]: reached home +[INFO] [1735525572.184999]: reset action history +[INFO] [1735525574.892768]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735525577.582033]: resetting environment +[INFO] [1735525577.584325]: cleared current plan +[INFO] [1735525577.584478]: moving to home +[INFO] [1735525578.585516]: reached home +[INFO] [1735525578.585939]: reset action history +[INFO] [1735525581.294039]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/episode_rosbags/episode_1_2024-12-29-21-26-21.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1360290.log +started roslaunch server http://158.130.50.37:44311/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1360713] +process[print_text-5]: started with pid [1360714] +process[rosbag_record-6]: started with pid [1360715] +started bagging! +[INFO] [1735525581.965423]: received mask_obs with header seq: 134 +stamp: + secs: 1735525581 + nsecs: 965919971 +frame_id: '' and grasped_object_mask header seq: 65402 +stamp: + secs: 1735525581 + nsecs: 766875505 +frame_id: "camera_color_optical_frame" +[INFO] [1735525582.178207]: received mask_obs with header seq: 135 +stamp: + secs: 1735525582 + nsecs: 179563760 +frame_id: '' and grasped_object_mask header seq: 65408 +stamp: + secs: 1735525581 + nsecs: 966479540 +frame_id: "camera_color_optical_frame" +[INFO] [1735525582.377648]: received mask_obs with header seq: 136 +stamp: + secs: 1735525582 + nsecs: 379032373 +frame_id: '' and grasped_object_mask header seq: 65414 +stamp: + secs: 1735525582 + nsecs: 166072845 +frame_id: "camera_color_optical_frame" +[INFO] [1735525582.582311]: received mask_obs with header seq: 137 +stamp: + secs: 1735525582 + nsecs: 583446502 +frame_id: '' and grasped_object_mask header seq: 65420 +stamp: + secs: 1735525582 + nsecs: 365654707 +frame_id: "camera_color_optical_frame" +[INFO] [1735525582.785367]: received mask_obs with header seq: 138 +stamp: + secs: 1735525582 + nsecs: 786638021 +frame_id: '' and grasped_object_mask header seq: 65426 +stamp: + secs: 1735525582 + nsecs: 565271616 +frame_id: "camera_color_optical_frame" +[INFO] [1735525582.998170]: received mask_obs with header seq: 139 +stamp: + secs: 1735525582 + nsecs: 999255180 +frame_id: '' and grasped_object_mask header seq: 65432 +stamp: + secs: 1735525582 + nsecs: 764882326 +frame_id: "camera_color_optical_frame" +[INFO] [1735525583.199663]: received mask_obs with header seq: 140 +stamp: + secs: 1735525583 + nsecs: 201008796 +frame_id: '' and grasped_object_mask header seq: 65438 +stamp: + secs: 1735525582 + nsecs: 964480400 +frame_id: "camera_color_optical_frame" +[INFO] [1735525583.404152]: received mask_obs with header seq: 141 +stamp: + secs: 1735525583 + nsecs: 405297994 +frame_id: '' and grasped_object_mask header seq: 65444 +stamp: + secs: 1735525583 + nsecs: 164067268 +frame_id: "camera_color_optical_frame" +[INFO] [1735525583.598209]: received mask_obs with header seq: 142 +stamp: + secs: 1735525583 + nsecs: 599400758 +frame_id: '' and grasped_object_mask header seq: 65449 +stamp: + secs: 1735525583 + nsecs: 330381393 +frame_id: "camera_color_optical_frame" +[INFO] [1735525583.806355]: received mask_obs with header seq: 143 +stamp: + secs: 1735525583 + nsecs: 807604789 +frame_id: '' and grasped_object_mask header seq: 65456 +stamp: + secs: 1735525583 + nsecs: 563273191 +frame_id: "camera_color_optical_frame" +[INFO] [1735525584.002249]: received mask_obs with header seq: 144 +stamp: + secs: 1735525584 + nsecs: 3598451 +frame_id: '' and grasped_object_mask header seq: 65463 +stamp: + secs: 1735525583 + nsecs: 796154976 +frame_id: "camera_color_optical_frame" +[INFO] [1735525584.204244]: received mask_obs with header seq: 145 +stamp: + secs: 1735525584 + nsecs: 205613851 +frame_id: '' and grasped_object_mask header seq: 65468 +stamp: + secs: 1735525583 + nsecs: 962490320 +frame_id: "camera_color_optical_frame" +[INFO] [1735525584.399089]: received mask_obs with header seq: 146 +stamp: + secs: 1735525584 + nsecs: 400189161 +frame_id: '' and grasped_object_mask header seq: 65474 +stamp: + secs: 1735525584 + nsecs: 162081480 +frame_id: "camera_color_optical_frame" +[INFO] [1735525584.603980]: received mask_obs with header seq: 147 +stamp: + secs: 1735525584 + nsecs: 605345249 +frame_id: '' and grasped_object_mask header seq: 65481 +stamp: + secs: 1735525584 + nsecs: 394923449 +frame_id: "camera_color_optical_frame" +[INFO] [1735525584.800972]: received mask_obs with header seq: 148 +stamp: + secs: 1735525584 + nsecs: 802096366 +frame_id: '' and grasped_object_mask header seq: 65487 +stamp: + secs: 1735525584 + nsecs: 594544888 +frame_id: "camera_color_optical_frame" +[INFO] [1735525585.001084]: received mask_obs with header seq: 149 +stamp: + secs: 1735525585 + nsecs: 2285242 +frame_id: '' and grasped_object_mask header seq: 65493 +stamp: + secs: 1735525584 + nsecs: 794153690 +frame_id: "camera_color_optical_frame" +[INFO] [1735525585.205352]: received mask_obs with header seq: 150 +stamp: + secs: 1735525585 + nsecs: 206449747 +frame_id: '' and grasped_object_mask header seq: 65499 +stamp: + secs: 1735525584 + nsecs: 993751526 +frame_id: "camera_color_optical_frame" +[INFO] [1735525585.403140]: received mask_obs with header seq: 151 +stamp: + secs: 1735525585 + nsecs: 404504060 +frame_id: '' and grasped_object_mask header seq: 65505 +stamp: + secs: 1735525585 + nsecs: 193337679 +frame_id: "camera_color_optical_frame" +[INFO] [1735525585.607717]: received mask_obs with header seq: 152 +stamp: + secs: 1735525585 + nsecs: 608429193 +frame_id: '' and grasped_object_mask header seq: 65511 +stamp: + secs: 1735525585 + nsecs: 392912388 +frame_id: "camera_color_optical_frame" +[INFO] [1735525585.804787]: received mask_obs with header seq: 153 +stamp: + secs: 1735525585 + nsecs: 805948972 +frame_id: '' and grasped_object_mask header seq: 65517 +stamp: + secs: 1735525585 + nsecs: 592534065 +frame_id: "camera_color_optical_frame" +[INFO] [1735525586.004151]: received mask_obs with header seq: 154 +stamp: + secs: 1735525586 + nsecs: 5481958 +frame_id: '' and grasped_object_mask header seq: 65523 +stamp: + secs: 1735525585 + nsecs: 792145729 +frame_id: "camera_color_optical_frame" +[INFO] [1735525586.202469]: received mask_obs with header seq: 155 +stamp: + secs: 1735525586 + nsecs: 203798294 +frame_id: '' and grasped_object_mask header seq: 65529 +stamp: + secs: 1735525585 + nsecs: 991745234 +frame_id: "camera_color_optical_frame" +[INFO] [1735525586.402130]: received mask_obs with header seq: 156 +stamp: + secs: 1735525586 + nsecs: 403232336 +frame_id: '' and grasped_object_mask header seq: 65535 +stamp: + secs: 1735525586 + nsecs: 191333294 +frame_id: "camera_color_optical_frame" +[INFO] [1735525586.608650]: received mask_obs with header seq: 157 +stamp: + secs: 1735525586 + nsecs: 608358144 +frame_id: '' and grasped_object_mask header seq: 65541 +stamp: + secs: 1735525586 + nsecs: 390910149 +frame_id: "camera_color_optical_frame" +[INFO] [1735525586.803364]: received mask_obs with header seq: 158 +stamp: + secs: 1735525586 + nsecs: 804805517 +frame_id: '' and grasped_object_mask header seq: 65547 +stamp: + secs: 1735525586 + nsecs: 590522289 +frame_id: "camera_color_optical_frame" +[INFO] [1735525587.005739]: received mask_obs with header seq: 159 +stamp: + secs: 1735525587 + nsecs: 6852626 +frame_id: '' and grasped_object_mask header seq: 65553 +stamp: + secs: 1735525586 + nsecs: 790122271 +frame_id: "camera_color_optical_frame" +[INFO] [1735525587.202373]: received mask_obs with header seq: 160 +stamp: + secs: 1735525587 + nsecs: 203735113 +frame_id: '' and grasped_object_mask header seq: 65559 +stamp: + secs: 1735525586 + nsecs: 989712000 +frame_id: "camera_color_optical_frame" +[INFO] [1735525587.404123]: received mask_obs with header seq: 161 +stamp: + secs: 1735525587 + nsecs: 405494451 +frame_id: '' and grasped_object_mask header seq: 65565 +stamp: + secs: 1735525587 + nsecs: 189288378 +frame_id: "camera_color_optical_frame" +[INFO] [1735525587.606338]: received mask_obs with header seq: 162 +stamp: + secs: 1735525587 + nsecs: 606475353 +frame_id: '' and grasped_object_mask header seq: 65571 +stamp: + secs: 1735525587 + nsecs: 388853312 +frame_id: "camera_color_optical_frame" +[INFO] [1735525587.803765]: received mask_obs with header seq: 163 +stamp: + secs: 1735525587 + nsecs: 804789304 +frame_id: '' and grasped_object_mask header seq: 65577 +stamp: + secs: 1735525587 + nsecs: 588467598 +frame_id: "camera_color_optical_frame" +[INFO] [1735525588.003449]: received mask_obs with header seq: 164 +stamp: + secs: 1735525588 + nsecs: 4785776 +frame_id: '' and grasped_object_mask header seq: 65583 +stamp: + secs: 1735525587 + nsecs: 788069963 +frame_id: "camera_color_optical_frame" +[INFO] [1735525588.204272]: received mask_obs with header seq: 165 +stamp: + secs: 1735525588 + nsecs: 205129146 +frame_id: '' and grasped_object_mask header seq: 65589 +stamp: + secs: 1735525587 + nsecs: 987660885 +frame_id: "camera_color_optical_frame" +[INFO] [1735525588.403730]: received mask_obs with header seq: 166 +stamp: + secs: 1735525588 + nsecs: 404737710 +frame_id: '' and grasped_object_mask header seq: 65595 +stamp: + secs: 1735525588 + nsecs: 187241316 +frame_id: "camera_color_optical_frame" +[INFO] [1735525588.610644]: received mask_obs with header seq: 167 +stamp: + secs: 1735525588 + nsecs: 611978054 +frame_id: '' and grasped_object_mask header seq: 65601 +stamp: + secs: 1735525588 + nsecs: 386808634 +frame_id: "camera_color_optical_frame" +[INFO] [1735525588.806917]: received mask_obs with header seq: 168 +stamp: + secs: 1735525588 + nsecs: 808212518 +frame_id: '' and grasped_object_mask header seq: 65607 +stamp: + secs: 1735525588 + nsecs: 588106394 +frame_id: "camera_color_optical_frame" +[INFO] [1735525589.006059]: received mask_obs with header seq: 169 +stamp: + secs: 1735525589 + nsecs: 5647420 +frame_id: '' and grasped_object_mask header seq: 65613 +stamp: + secs: 1735525588 + nsecs: 790051460 +frame_id: "camera_color_optical_frame" +[INFO] [1735525589.201664]: received mask_obs with header seq: 170 +stamp: + secs: 1735525589 + nsecs: 203059196 +frame_id: '' and grasped_object_mask header seq: 65619 +stamp: + secs: 1735525588 + nsecs: 992070913 +frame_id: "camera_color_optical_frame" +[INFO] [1735525589.401170]: received mask_obs with header seq: 171 +stamp: + secs: 1735525589 + nsecs: 402530908 +frame_id: '' and grasped_object_mask header seq: 65625 +stamp: + secs: 1735525589 + nsecs: 194164515 +frame_id: "camera_color_optical_frame" +[INFO] [1735525589.604748]: received mask_obs with header seq: 172 +stamp: + secs: 1735525589 + nsecs: 605421066 +frame_id: '' and grasped_object_mask header seq: 65631 +stamp: + secs: 1735525589 + nsecs: 396333694 +frame_id: "camera_color_optical_frame" +[INFO] [1735525589.802621]: received mask_obs with header seq: 173 +stamp: + secs: 1735525589 + nsecs: 803645610 +frame_id: '' and grasped_object_mask header seq: 65637 +stamp: + secs: 1735525589 + nsecs: 596484900 +frame_id: "camera_color_optical_frame" +[INFO] [1735525590.002298]: received mask_obs with header seq: 174 +stamp: + secs: 1735525590 + nsecs: 3656864 +frame_id: '' and grasped_object_mask header seq: 65643 +stamp: + secs: 1735525589 + nsecs: 795868158 +frame_id: "camera_color_optical_frame" +[INFO] [1735525590.202058]: received mask_obs with header seq: 175 +stamp: + secs: 1735525590 + nsecs: 203429937 +frame_id: '' and grasped_object_mask header seq: 65649 +stamp: + secs: 1735525589 + nsecs: 995228767 +frame_id: "camera_color_optical_frame" +[INFO] [1735525590.405619]: received mask_obs with header seq: 176 +stamp: + secs: 1735525590 + nsecs: 406923532 +frame_id: '' and grasped_object_mask header seq: 65655 +stamp: + secs: 1735525590 + nsecs: 194566488 +frame_id: "camera_color_optical_frame" +[INFO] [1735525590.604913]: received mask_obs with header seq: 177 +stamp: + secs: 1735525590 + nsecs: 605347633 +frame_id: '' and grasped_object_mask header seq: 65661 +stamp: + secs: 1735525590 + nsecs: 393881083 +frame_id: "camera_color_optical_frame" +[INFO] [1735525590.804903]: received mask_obs with header seq: 178 +stamp: + secs: 1735525590 + nsecs: 805373191 +frame_id: '' and grasped_object_mask header seq: 65667 +stamp: + secs: 1735525590 + nsecs: 594174385 +frame_id: "camera_color_optical_frame" +[INFO] [1735525591.001279]: received mask_obs with header seq: 179 +stamp: + secs: 1735525591 + nsecs: 2637147 +frame_id: '' and grasped_object_mask header seq: 65673 +stamp: + secs: 1735525590 + nsecs: 794898033 +frame_id: "camera_color_optical_frame" +[INFO] [1735525591.200998]: received mask_obs with header seq: 180 +stamp: + secs: 1735525591 + nsecs: 202343463 +frame_id: '' and grasped_object_mask header seq: 65679 +stamp: + secs: 1735525590 + nsecs: 995695114 +frame_id: "camera_color_optical_frame" +[INFO] [1735525591.401455]: received mask_obs with header seq: 181 +stamp: + secs: 1735525591 + nsecs: 402778148 +frame_id: '' and grasped_object_mask header seq: 65685 +stamp: + secs: 1735525591 + nsecs: 196566582 +frame_id: "camera_color_optical_frame" +[INFO] [1735525591.603842]: received mask_obs with header seq: 182 +stamp: + secs: 1735525591 + nsecs: 604758024 +frame_id: '' and grasped_object_mask header seq: 65691 +stamp: + secs: 1735525591 + nsecs: 397512913 +frame_id: "camera_color_optical_frame" +[INFO] [1735525591.799954]: received mask_obs with header seq: 183 +stamp: + secs: 1735525591 + nsecs: 801372766 +frame_id: '' and grasped_object_mask header seq: 65697 +stamp: + secs: 1735525591 + nsecs: 597503662 +frame_id: "camera_color_optical_frame" +[INFO] [1735525592.004002]: received mask_obs with header seq: 184 +stamp: + secs: 1735525592 + nsecs: 2919673 +frame_id: '' and grasped_object_mask header seq: 65703 +stamp: + secs: 1735525591 + nsecs: 797111034 +frame_id: "camera_color_optical_frame" +[INFO] [1735525592.201178]: received mask_obs with header seq: 185 +stamp: + secs: 1735525592 + nsecs: 202354431 +frame_id: '' and grasped_object_mask header seq: 65709 +stamp: + secs: 1735525591 + nsecs: 996708155 +frame_id: "camera_color_optical_frame" +[INFO] [1735525592.402962]: received mask_obs with header seq: 186 +stamp: + secs: 1735525592 + nsecs: 402921676 +frame_id: '' and grasped_object_mask header seq: 65715 +stamp: + secs: 1735525592 + nsecs: 196293116 +frame_id: "camera_color_optical_frame" +[INFO] [1735525592.604465]: received mask_obs with header seq: 187 +stamp: + secs: 1735525592 + nsecs: 604525089 +frame_id: '' and grasped_object_mask header seq: 65721 +stamp: + secs: 1735525592 + nsecs: 395866632 +frame_id: "camera_color_optical_frame" +[INFO] [1735525592.801208]: received mask_obs with header seq: 188 +stamp: + secs: 1735525592 + nsecs: 802202463 +frame_id: '' and grasped_object_mask header seq: 65727 +stamp: + secs: 1735525592 + nsecs: 595469475 +frame_id: "camera_color_optical_frame" +[INFO] [1735525593.003191]: received mask_obs with header seq: 189 +stamp: + secs: 1735525593 + nsecs: 3825902 +frame_id: '' and grasped_object_mask header seq: 65733 +stamp: + secs: 1735525592 + nsecs: 795076370 +frame_id: "camera_color_optical_frame" +[INFO] [1735525593.201654]: received mask_obs with header seq: 190 +stamp: + secs: 1735525593 + nsecs: 202753543 +frame_id: '' and grasped_object_mask header seq: 65739 +stamp: + secs: 1735525592 + nsecs: 994671583 +frame_id: "camera_color_optical_frame" +[INFO] [1735525593.402313]: received mask_obs with header seq: 191 +stamp: + secs: 1735525593 + nsecs: 402771472 +frame_id: '' and grasped_object_mask header seq: 65745 +stamp: + secs: 1735525593 + nsecs: 194256067 +frame_id: "camera_color_optical_frame" +[INFO] [1735525593.605936]: received mask_obs with header seq: 192 +stamp: + secs: 1735525593 + nsecs: 607293605 +frame_id: '' and grasped_object_mask header seq: 65751 +stamp: + secs: 1735525593 + nsecs: 393828392 +frame_id: "camera_color_optical_frame" +[INFO] [1735525593.801293]: received mask_obs with header seq: 193 +stamp: + secs: 1735525593 + nsecs: 802085161 +frame_id: '' and grasped_object_mask header seq: 65757 +stamp: + secs: 1735525593 + nsecs: 593433857 +frame_id: "camera_color_optical_frame" +[INFO] [1735525594.002042]: received mask_obs with header seq: 194 +stamp: + secs: 1735525594 + nsecs: 3313779 +frame_id: '' and grasped_object_mask header seq: 65763 +stamp: + secs: 1735525593 + nsecs: 793045282 +frame_id: "camera_color_optical_frame" +[INFO] [1735525594.201415]: received mask_obs with header seq: 195 +stamp: + secs: 1735525594 + nsecs: 201129913 +frame_id: '' and grasped_object_mask header seq: 65769 +stamp: + secs: 1735525593 + nsecs: 992645264 +frame_id: "camera_color_optical_frame" +[INFO] [1735525594.402148]: received mask_obs with header seq: 196 +stamp: + secs: 1735525594 + nsecs: 402142524 +frame_id: '' and grasped_object_mask header seq: 65775 +stamp: + secs: 1735525594 + nsecs: 192233801 +frame_id: "camera_color_optical_frame" +[INFO] [1735525594.609006]: received mask_obs with header seq: 197 +stamp: + secs: 1735525594 + nsecs: 610191822 +frame_id: '' and grasped_object_mask header seq: 65781 +stamp: + secs: 1735525594 + nsecs: 391812325 +frame_id: "camera_color_optical_frame" +[INFO] [1735525594.800933]: received mask_obs with header seq: 198 +stamp: + secs: 1735525594 + nsecs: 801984786 +frame_id: '' and grasped_object_mask header seq: 65787 +stamp: + secs: 1735525594 + nsecs: 591421366 +frame_id: "camera_color_optical_frame" +[INFO] [1735525595.001554]: received mask_obs with header seq: 199 +stamp: + secs: 1735525595 + nsecs: 2250194 +frame_id: '' and grasped_object_mask header seq: 65793 +stamp: + secs: 1735525594 + nsecs: 791035652 +frame_id: "camera_color_optical_frame" +[INFO] [1735525595.200682]: received mask_obs with header seq: 200 +stamp: + secs: 1735525595 + nsecs: 201296091 +frame_id: '' and grasped_object_mask header seq: 65799 +stamp: + secs: 1735525594 + nsecs: 990638971 +frame_id: "camera_color_optical_frame" +[INFO] [1735525595.406505]: received mask_obs with header seq: 201 +stamp: + secs: 1735525595 + nsecs: 407649755 +frame_id: '' and grasped_object_mask header seq: 65805 +stamp: + secs: 1735525595 + nsecs: 190230608 +frame_id: "camera_color_optical_frame" +[INFO] [1735525595.603292]: received mask_obs with header seq: 202 +stamp: + secs: 1735525595 + nsecs: 604236364 +frame_id: '' and grasped_object_mask header seq: 65811 +stamp: + secs: 1735525595 + nsecs: 389811277 +frame_id: "camera_color_optical_frame" +[INFO] [1735525595.800704]: received mask_obs with header seq: 203 +stamp: + secs: 1735525595 + nsecs: 800305604 +frame_id: '' and grasped_object_mask header seq: 65817 +stamp: + secs: 1735525595 + nsecs: 589421272 +frame_id: "camera_color_optical_frame" +[INFO] [1735525596.001191]: received mask_obs with header seq: 204 +stamp: + secs: 1735525596 + nsecs: 2134323 +frame_id: '' and grasped_object_mask header seq: 65823 +stamp: + secs: 1735525595 + nsecs: 789025068 +frame_id: "camera_color_optical_frame" +[INFO] [1735525596.200723]: received mask_obs with header seq: 205 +stamp: + secs: 1735525596 + nsecs: 201400756 +frame_id: '' and grasped_object_mask header seq: 65829 +stamp: + secs: 1735525595 + nsecs: 988617182 +frame_id: "camera_color_optical_frame" +[INFO] [1735525596.399839]: received mask_obs with header seq: 206 +stamp: + secs: 1735525596 + nsecs: 400627374 +frame_id: '' and grasped_object_mask header seq: 65835 +stamp: + secs: 1735525596 + nsecs: 188197374 +frame_id: "camera_color_optical_frame" +[INFO] [1735525596.602221]: received mask_obs with header seq: 207 +stamp: + secs: 1735525596 + nsecs: 603593111 +frame_id: '' and grasped_object_mask header seq: 65841 +stamp: + secs: 1735525596 + nsecs: 387765646 +frame_id: "camera_color_optical_frame" +[INFO] [1735525596.799315]: received mask_obs with header seq: 208 +stamp: + secs: 1735525596 + nsecs: 799669981 +frame_id: '' and grasped_object_mask header seq: 65847 +stamp: + secs: 1735525596 + nsecs: 587366104 +frame_id: "camera_color_optical_frame" +[INFO] [1735525597.000128]: received mask_obs with header seq: 209 +stamp: + secs: 1735525596 + nsecs: 999657392 +frame_id: '' and grasped_object_mask header seq: 65853 +stamp: + secs: 1735525596 + nsecs: 786978245 +frame_id: "camera_color_optical_frame" +[INFO] [1735525597.204059]: received mask_obs with header seq: 210 +stamp: + secs: 1735525597 + nsecs: 205078125 +frame_id: '' and grasped_object_mask header seq: 65859 +stamp: + secs: 1735525596 + nsecs: 986577988 +frame_id: "camera_color_optical_frame" +[INFO] [1735525597.399222]: received mask_obs with header seq: 211 +stamp: + secs: 1735525597 + nsecs: 400573492 +frame_id: '' and grasped_object_mask header seq: 65865 +stamp: + secs: 1735525597 + nsecs: 186166286 +frame_id: "camera_color_optical_frame" +[INFO] [1735525597.602359]: received mask_obs with header seq: 212 +stamp: + secs: 1735525597 + nsecs: 603350162 +frame_id: '' and grasped_object_mask header seq: 65871 +stamp: + secs: 1735525597 + nsecs: 385743141 +frame_id: "camera_color_optical_frame" +[INFO] [1735525597.799030]: received mask_obs with header seq: 213 +stamp: + secs: 1735525597 + nsecs: 799861907 +frame_id: '' and grasped_object_mask header seq: 65877 +stamp: + secs: 1735525597 + nsecs: 585344076 +frame_id: "camera_color_optical_frame" +[INFO] [1735525597.999614]: received mask_obs with header seq: 214 +stamp: + secs: 1735525597 + nsecs: 999475002 +frame_id: '' and grasped_object_mask header seq: 65883 +stamp: + secs: 1735525597 + nsecs: 784951448 +frame_id: "camera_color_optical_frame" +[INFO] [1735525598.198920]: received mask_obs with header seq: 215 +stamp: + secs: 1735525598 + nsecs: 199253559 +frame_id: '' and grasped_object_mask header seq: 65889 +stamp: + secs: 1735525597 + nsecs: 984547138 +frame_id: "camera_color_optical_frame" +[INFO] [1735525598.398615]: received mask_obs with header seq: 216 +stamp: + secs: 1735525598 + nsecs: 399179458 +frame_id: '' and grasped_object_mask header seq: 65895 +stamp: + secs: 1735525598 + nsecs: 184132576 +frame_id: "camera_color_optical_frame" +[INFO] [1735525598.601802]: received mask_obs with header seq: 217 +stamp: + secs: 1735525598 + nsecs: 602777957 +frame_id: '' and grasped_object_mask header seq: 65901 +stamp: + secs: 1735525598 + nsecs: 383705378 +frame_id: "camera_color_optical_frame" +[INFO] [1735525598.798395]: received mask_obs with header seq: 218 +stamp: + secs: 1735525598 + nsecs: 798213720 +frame_id: '' and grasped_object_mask header seq: 65907 +stamp: + secs: 1735525598 + nsecs: 583306074 +frame_id: "camera_color_optical_frame" +[INFO] [1735525598.998424]: received mask_obs with header seq: 219 +stamp: + secs: 1735525598 + nsecs: 999169588 +frame_id: '' and grasped_object_mask header seq: 65913 +stamp: + secs: 1735525598 + nsecs: 782915831 +frame_id: "camera_color_optical_frame" +[INFO] [1735525599.198616]: received mask_obs with header seq: 220 +stamp: + secs: 1735525599 + nsecs: 199861764 +frame_id: '' and grasped_object_mask header seq: 65919 +stamp: + secs: 1735525598 + nsecs: 982513666 +frame_id: "camera_color_optical_frame" +[INFO] [1735525599.404773]: received mask_obs with header seq: 221 +stamp: + secs: 1735525599 + nsecs: 405682802 +frame_id: '' and grasped_object_mask header seq: 65925 +stamp: + secs: 1735525599 + nsecs: 182100058 +frame_id: "camera_color_optical_frame" +[INFO] [1735525599.606021]: received mask_obs with header seq: 222 +stamp: + secs: 1735525599 + nsecs: 607428073 +frame_id: '' and grasped_object_mask header seq: 65931 +stamp: + secs: 1735525599 + nsecs: 381675959 +frame_id: "camera_color_optical_frame" +[INFO] [1735525599.797593]: received mask_obs with header seq: 223 +stamp: + secs: 1735525599 + nsecs: 797922372 +frame_id: '' and grasped_object_mask header seq: 65937 +stamp: + secs: 1735525599 + nsecs: 581279755 +frame_id: "camera_color_optical_frame" +[INFO] [1735525599.999351]: received mask_obs with header seq: 224 +stamp: + secs: 1735525599 + nsecs: 997482299 +frame_id: '' and grasped_object_mask header seq: 65943 +stamp: + secs: 1735525599 + nsecs: 780892372 +frame_id: "camera_color_optical_frame" +[INFO] [1735525600.197900]: received mask_obs with header seq: 225 +stamp: + secs: 1735525600 + nsecs: 198395252 +frame_id: '' and grasped_object_mask header seq: 65949 +stamp: + secs: 1735525599 + nsecs: 980493307 +frame_id: "camera_color_optical_frame" +[INFO] [1735525600.396970]: received mask_obs with header seq: 226 +stamp: + secs: 1735525600 + nsecs: 398371219 +frame_id: '' and grasped_object_mask header seq: 65955 +stamp: + secs: 1735525600 + nsecs: 180083036 +frame_id: "camera_color_optical_frame" +[INFO] [1735525600.599134]: received mask_obs with header seq: 227 +stamp: + secs: 1735525600 + nsecs: 600231647 +frame_id: '' and grasped_object_mask header seq: 65961 +stamp: + secs: 1735525600 + nsecs: 379661322 +frame_id: "camera_color_optical_frame" +[INFO] [1735525600.796040]: received mask_obs with header seq: 228 +stamp: + secs: 1735525600 + nsecs: 796160936 +frame_id: '' and grasped_object_mask header seq: 65967 +stamp: + secs: 1735525600 + nsecs: 579267263 +frame_id: "camera_color_optical_frame" +[INFO] [1735525600.996071]: received mask_obs with header seq: 229 +stamp: + secs: 1735525600 + nsecs: 997476339 +frame_id: '' and grasped_object_mask header seq: 65973 +stamp: + secs: 1735525600 + nsecs: 778879881 +frame_id: "camera_color_optical_frame" +[INFO] [1735525601.196430]: received mask_obs with header seq: 230 +stamp: + secs: 1735525601 + nsecs: 196130275 +frame_id: '' and grasped_object_mask header seq: 65979 +stamp: + secs: 1735525600 + nsecs: 978481293 +frame_id: "camera_color_optical_frame" +[INFO] [1735525601.400910]: received mask_obs with header seq: 231 +stamp: + secs: 1735525601 + nsecs: 401985645 +frame_id: '' and grasped_object_mask header seq: 65985 +stamp: + secs: 1735525601 + nsecs: 178071022 +frame_id: "camera_color_optical_frame" +[INFO] [1735525601.598444]: received mask_obs with header seq: 232 +stamp: + secs: 1735525601 + nsecs: 599491357 +frame_id: '' and grasped_object_mask header seq: 65991 +stamp: + secs: 1735525601 + nsecs: 377649546 +frame_id: "camera_color_optical_frame" +[INFO] [1735525601.796136]: received mask_obs with header seq: 233 +stamp: + secs: 1735525601 + nsecs: 796329021 +frame_id: '' and grasped_object_mask header seq: 65997 +stamp: + secs: 1735525601 + nsecs: 577251434 +frame_id: "camera_color_optical_frame" +[INFO] [1735525601.995654]: received mask_obs with header seq: 234 +stamp: + secs: 1735525601 + nsecs: 996458768 +frame_id: '' and grasped_object_mask header seq: 66003 +stamp: + secs: 1735525601 + nsecs: 776859760 +frame_id: "camera_color_optical_frame" +[INFO] [1735525602.195767]: received mask_obs with header seq: 235 +stamp: + secs: 1735525602 + nsecs: 196643352 +frame_id: '' and grasped_object_mask header seq: 66009 +stamp: + secs: 1735525601 + nsecs: 976455927 +frame_id: "camera_color_optical_frame" +[INFO] [1735525602.396088]: received mask_obs with header seq: 236 +stamp: + secs: 1735525602 + nsecs: 397348165 +frame_id: '' and grasped_object_mask header seq: 66015 +stamp: + secs: 1735525602 + nsecs: 176039934 +frame_id: "camera_color_optical_frame" +[INFO] [1735525602.598805]: received mask_obs with header seq: 237 +stamp: + secs: 1735525602 + nsecs: 600227117 +frame_id: '' and grasped_object_mask header seq: 66021 +stamp: + secs: 1735525602 + nsecs: 375612497 +frame_id: "camera_color_optical_frame" +[INFO] [1735525602.795301]: received mask_obs with header seq: 238 +stamp: + secs: 1735525602 + nsecs: 795889854 +frame_id: '' and grasped_object_mask header seq: 66027 +stamp: + secs: 1735525602 + nsecs: 575210094 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -107331095 for nearest: 1735525602667483511 - target: 1735525602774814606 at idx 249 +[INFO] [1735525602.997249]: received mask_obs with header seq: 239 +stamp: + secs: 1735525602 + nsecs: 998633384 +frame_id: '' and grasped_object_mask header seq: 66033 +stamp: + secs: 1735525602 + nsecs: 774814606 +frame_id: "camera_color_optical_frame" +[INFO] [1735525603.195598]: received mask_obs with header seq: 240 +stamp: + secs: 1735525603 + nsecs: 195858001 +frame_id: '' and grasped_object_mask header seq: 66039 +stamp: + secs: 1735525602 + nsecs: 974408150 +frame_id: "camera_color_optical_frame" +[INFO] [1735525603.399257]: received mask_obs with header seq: 241 +stamp: + secs: 1735525603 + nsecs: 400171995 +frame_id: '' and grasped_object_mask header seq: 66045 +stamp: + secs: 1735525603 + nsecs: 173988819 +frame_id: "camera_color_optical_frame" +[INFO] [1735525603.597250]: received mask_obs with header seq: 242 +stamp: + secs: 1735525603 + nsecs: 598447084 +frame_id: '' and grasped_object_mask header seq: 66051 +stamp: + secs: 1735525603 + nsecs: 373557329 +frame_id: "camera_color_optical_frame" +[INFO] [1735525603.796774]: received mask_obs with header seq: 243 +stamp: + secs: 1735525603 + nsecs: 798171043 +frame_id: '' and grasped_object_mask header seq: 66057 +stamp: + secs: 1735525603 + nsecs: 573157310 +frame_id: "camera_color_optical_frame" +[INFO] [1735525604.005450]: received mask_obs with header seq: 244 +stamp: + secs: 1735525604 + nsecs: 6691694 +frame_id: '' and grasped_object_mask header seq: 66063 +stamp: + secs: 1735525603 + nsecs: 772766590 +frame_id: "camera_color_optical_frame" +[INFO] [1735525604.204646]: received mask_obs with header seq: 245 +stamp: + secs: 1735525604 + nsecs: 205808639 +frame_id: '' and grasped_object_mask header seq: 66069 +stamp: + secs: 1735525603 + nsecs: 972364426 +frame_id: "camera_color_optical_frame" +[INFO] [1735525604.397288]: received mask_obs with header seq: 246 +stamp: + secs: 1735525604 + nsecs: 398296356 +frame_id: '' and grasped_object_mask header seq: 66075 +stamp: + secs: 1735525604 + nsecs: 171951771 +frame_id: "camera_color_optical_frame" +[INFO] [1735525604.597704]: received mask_obs with header seq: 247 +stamp: + secs: 1735525604 + nsecs: 599016666 +frame_id: '' and grasped_object_mask header seq: 66081 +stamp: + secs: 1735525604 + nsecs: 371526241 +frame_id: "camera_color_optical_frame" +[INFO] [1735525604.796534]: received mask_obs with header seq: 248 +stamp: + secs: 1735525604 + nsecs: 797809839 +frame_id: '' and grasped_object_mask header seq: 66087 +stamp: + secs: 1735525604 + nsecs: 571961880 +frame_id: "camera_color_optical_frame" +[INFO] [1735525604.996489]: received mask_obs with header seq: 249 +stamp: + secs: 1735525604 + nsecs: 997789859 +frame_id: '' and grasped_object_mask header seq: 66093 +stamp: + secs: 1735525604 + nsecs: 772904396 +frame_id: "camera_color_optical_frame" +[INFO] [1735525605.195682]: received mask_obs with header seq: 250 +stamp: + secs: 1735525605 + nsecs: 197063446 +frame_id: '' and grasped_object_mask header seq: 66099 +stamp: + secs: 1735525604 + nsecs: 973933458 +frame_id: "camera_color_optical_frame" +[INFO] [1735525605.396423]: received mask_obs with header seq: 251 +stamp: + secs: 1735525605 + nsecs: 397531032 +frame_id: '' and grasped_object_mask header seq: 66105 +stamp: + secs: 1735525605 + nsecs: 175048828 +frame_id: "camera_color_optical_frame" +[INFO] [1735525605.597622]: received mask_obs with header seq: 252 +stamp: + secs: 1735525605 + nsecs: 598976850 +frame_id: '' and grasped_object_mask header seq: 66111 +stamp: + secs: 1735525605 + nsecs: 376250982 +frame_id: "camera_color_optical_frame" +[INFO] [1735525605.795489]: received mask_obs with header seq: 253 +stamp: + secs: 1735525605 + nsecs: 796871423 +frame_id: '' and grasped_object_mask header seq: 66117 +stamp: + secs: 1735525605 + nsecs: 576566458 +frame_id: "camera_color_optical_frame" +[INFO] [1735525605.995918]: received mask_obs with header seq: 254 +stamp: + secs: 1735525605 + nsecs: 997081279 +frame_id: '' and grasped_object_mask header seq: 66123 +stamp: + secs: 1735525605 + nsecs: 776408672 +frame_id: "camera_color_optical_frame" +[INFO] [1735525606.195539]: received mask_obs with header seq: 255 +stamp: + secs: 1735525606 + nsecs: 196898221 +frame_id: '' and grasped_object_mask header seq: 66129 +stamp: + secs: 1735525605 + nsecs: 976251125 +frame_id: "camera_color_optical_frame" +[INFO] [1735525606.394890]: received mask_obs with header seq: 256 +stamp: + secs: 1735525606 + nsecs: 396278619 +frame_id: '' and grasped_object_mask header seq: 66135 +stamp: + secs: 1735525606 + nsecs: 176094055 +frame_id: "camera_color_optical_frame" +[INFO] [1735525606.598840]: received mask_obs with header seq: 257 +stamp: + secs: 1735525606 + nsecs: 600193977 +frame_id: '' and grasped_object_mask header seq: 66141 +stamp: + secs: 1735525606 + nsecs: 375936985 +frame_id: "camera_color_optical_frame" +[INFO] [1735525606.797120]: received mask_obs with header seq: 258 +stamp: + secs: 1735525606 + nsecs: 797932147 +frame_id: '' and grasped_object_mask header seq: 66147 +stamp: + secs: 1735525606 + nsecs: 575776339 +frame_id: "camera_color_optical_frame" +[INFO] [1735525606.997660]: received mask_obs with header seq: 259 +stamp: + secs: 1735525606 + nsecs: 999050378 +frame_id: '' and grasped_object_mask header seq: 66153 +stamp: + secs: 1735525606 + nsecs: 775614500 +frame_id: "camera_color_optical_frame" +[INFO] [1735525607.196963]: received mask_obs with header seq: 260 +stamp: + secs: 1735525607 + nsecs: 198260307 +frame_id: '' and grasped_object_mask header seq: 66159 +stamp: + secs: 1735525606 + nsecs: 975451469 +frame_id: "camera_color_optical_frame" +[INFO] [1735525607.396824]: received mask_obs with header seq: 261 +stamp: + secs: 1735525607 + nsecs: 398066282 +frame_id: '' and grasped_object_mask header seq: 66165 +stamp: + secs: 1735525607 + nsecs: 175288677 +frame_id: "camera_color_optical_frame" +[INFO] [1735525607.597751]: received mask_obs with header seq: 262 +stamp: + secs: 1735525607 + nsecs: 599151849 +frame_id: '' and grasped_object_mask header seq: 66171 +stamp: + secs: 1735525607 + nsecs: 375125647 +frame_id: "camera_color_optical_frame" +[INFO] [1735525607.797655]: received mask_obs with header seq: 263 +stamp: + secs: 1735525607 + nsecs: 799041509 +frame_id: '' and grasped_object_mask header seq: 66177 +stamp: + secs: 1735525607 + nsecs: 574961424 +frame_id: "camera_color_optical_frame" +[INFO] [1735525608.004800]: received mask_obs with header seq: 264 +stamp: + secs: 1735525608 + nsecs: 5894422 +frame_id: '' and grasped_object_mask header seq: 66183 +stamp: + secs: 1735525607 + nsecs: 774796247 +frame_id: "camera_color_optical_frame" +[INFO] [1735525608.204695]: received mask_obs with header seq: 265 +stamp: + secs: 1735525608 + nsecs: 205921649 +frame_id: '' and grasped_object_mask header seq: 66189 +stamp: + secs: 1735525607 + nsecs: 974631786 +frame_id: "camera_color_optical_frame" +[INFO] [1735525608.245866]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1735525608.395360]: received mask_obs with header seq: 266 +stamp: + secs: 1735525608 + nsecs: 396772146 +frame_id: '' and grasped_object_mask header seq: 66195 +stamp: + secs: 1735525608 + nsecs: 174466133 +frame_id: "camera_color_optical_frame" +[INFO] [1735525609.016948]: Stored episode 2. +[INFO] [1735525609.017624]: Saving video... +[WARN] [1735525609.579184]: Plan exhausted +[WARN] [1735525609.618559]: Plan exhausted +[WARN] [1735525609.658551]: Plan exhausted +[WARN] [1735525609.701074]: Plan exhausted +[INFO] [1735525609.642464]: Video saved! +Starting episode 2 +[INFO] [1735525609.698537]: resetting environment +[INFO] [1735525609.707926]: cleared current plan +[INFO] [1735525609.708495]: moving to home +[INFO] [1735525613.318638]: reached home +[INFO] [1735525613.320322]: reset action history +[INFO] [1735525616.090857]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735525618.397297]: resetting environment +[INFO] [1735525618.400971]: cleared current plan +[INFO] [1735525618.401273]: moving to home +[INFO] [1735525619.404743]: reached home +[INFO] [1735525619.405300]: reset action history +[INFO] [1735525622.116601]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/episode_rosbags/episode_2_2024-12-29-21-27-02.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1360290.log +started roslaunch server http://158.130.50.37:39903/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1360850] +process[print_text-8]: started with pid [1360851] +process[rosbag_record-9]: started with pid [1360875] +started bagging! +[INFO] [1735525622.820840]: received mask_obs with header seq: 267 +stamp: + secs: 1735525622 + nsecs: 821759939 +frame_id: '' and grasped_object_mask header seq: 66628 +stamp: + secs: 1735525622 + nsecs: 596168518 +frame_id: "camera_color_optical_frame" +[INFO] [1735525623.031767]: received mask_obs with header seq: 268 +stamp: + secs: 1735525623 + nsecs: 32862663 +frame_id: '' and grasped_object_mask header seq: 66635 +stamp: + secs: 1735525622 + nsecs: 829304218 +frame_id: "camera_color_optical_frame" +[INFO] [1735525623.224785]: received mask_obs with header seq: 269 +stamp: + secs: 1735525623 + nsecs: 226176023 +frame_id: '' and grasped_object_mask header seq: 66641 +stamp: + secs: 1735525623 + nsecs: 29134274 +frame_id: "camera_color_optical_frame" +[INFO] [1735525623.423498]: received mask_obs with header seq: 270 +stamp: + secs: 1735525623 + nsecs: 424714803 +frame_id: '' and grasped_object_mask header seq: 66646 +stamp: + secs: 1735525623 + nsecs: 195658445 +frame_id: "camera_color_optical_frame" +[INFO] [1735525623.634786]: received mask_obs with header seq: 271 +stamp: + secs: 1735525623 + nsecs: 636190891 +frame_id: '' and grasped_object_mask header seq: 66653 +stamp: + secs: 1735525623 + nsecs: 428793192 +frame_id: "camera_color_optical_frame" +[INFO] [1735525623.830548]: received mask_obs with header seq: 272 +stamp: + secs: 1735525623 + nsecs: 831881046 +frame_id: '' and grasped_object_mask header seq: 66659 +stamp: + secs: 1735525623 + nsecs: 628633976 +frame_id: "camera_color_optical_frame" +[INFO] [1735525624.032508]: received mask_obs with header seq: 273 +stamp: + secs: 1735525624 + nsecs: 33761262 +frame_id: '' and grasped_object_mask header seq: 66665 +stamp: + secs: 1735525623 + nsecs: 828477621 +frame_id: "camera_color_optical_frame" +[INFO] [1735525624.237229]: received mask_obs with header seq: 274 +stamp: + secs: 1735525624 + nsecs: 238450765 +frame_id: '' and grasped_object_mask header seq: 66671 +stamp: + secs: 1735525624 + nsecs: 28321981 +frame_id: "camera_color_optical_frame" +[INFO] [1735525624.441564]: received mask_obs with header seq: 275 +stamp: + secs: 1735525624 + nsecs: 442759752 +frame_id: '' and grasped_object_mask header seq: 66677 +stamp: + secs: 1735525624 + nsecs: 228166103 +frame_id: "camera_color_optical_frame" +[INFO] [1735525624.639064]: received mask_obs with header seq: 276 +stamp: + secs: 1735525624 + nsecs: 640467643 +frame_id: '' and grasped_object_mask header seq: 66683 +stamp: + secs: 1735525624 + nsecs: 428009987 +frame_id: "camera_color_optical_frame" +[INFO] [1735525624.839625]: received mask_obs with header seq: 277 +stamp: + secs: 1735525624 + nsecs: 840889453 +frame_id: '' and grasped_object_mask header seq: 66689 +stamp: + secs: 1735525624 + nsecs: 627847195 +frame_id: "camera_color_optical_frame" +[INFO] [1735525625.042622]: received mask_obs with header seq: 278 +stamp: + secs: 1735525625 + nsecs: 43111085 +frame_id: '' and grasped_object_mask header seq: 66695 +stamp: + secs: 1735525624 + nsecs: 827682257 +frame_id: "camera_color_optical_frame" +[INFO] [1735525625.240367]: received mask_obs with header seq: 279 +stamp: + secs: 1735525625 + nsecs: 241786718 +frame_id: '' and grasped_object_mask header seq: 66701 +stamp: + secs: 1735525625 + nsecs: 27517796 +frame_id: "camera_color_optical_frame" +[INFO] [1735525625.440796]: received mask_obs with header seq: 280 +stamp: + secs: 1735525625 + nsecs: 442188739 +frame_id: '' and grasped_object_mask header seq: 66707 +stamp: + secs: 1735525625 + nsecs: 227354050 +frame_id: "camera_color_optical_frame" +[INFO] [1735525625.639940]: received mask_obs with header seq: 281 +stamp: + secs: 1735525625 + nsecs: 641347408 +frame_id: '' and grasped_object_mask header seq: 66713 +stamp: + secs: 1735525625 + nsecs: 427188873 +frame_id: "camera_color_optical_frame" +[INFO] [1735525625.839486]: received mask_obs with header seq: 282 +stamp: + secs: 1735525625 + nsecs: 840879917 +frame_id: '' and grasped_object_mask header seq: 66719 +stamp: + secs: 1735525625 + nsecs: 627024174 +frame_id: "camera_color_optical_frame" +[INFO] [1735525626.039513]: received mask_obs with header seq: 283 +stamp: + secs: 1735525626 + nsecs: 40887594 +frame_id: '' and grasped_object_mask header seq: 66725 +stamp: + secs: 1735525625 + nsecs: 826859474 +frame_id: "camera_color_optical_frame" +[INFO] [1735525626.240211]: received mask_obs with header seq: 284 +stamp: + secs: 1735525626 + nsecs: 241509199 +frame_id: '' and grasped_object_mask header seq: 66731 +stamp: + secs: 1735525626 + nsecs: 26694298 +frame_id: "camera_color_optical_frame" +[INFO] [1735525626.440150]: received mask_obs with header seq: 285 +stamp: + secs: 1735525626 + nsecs: 441563367 +frame_id: '' and grasped_object_mask header seq: 66737 +stamp: + secs: 1735525626 + nsecs: 226528883 +frame_id: "camera_color_optical_frame" +[INFO] [1735525626.640623]: received mask_obs with header seq: 286 +stamp: + secs: 1735525626 + nsecs: 640092372 +frame_id: '' and grasped_object_mask header seq: 66743 +stamp: + secs: 1735525626 + nsecs: 426363945 +frame_id: "camera_color_optical_frame" +[INFO] [1735525626.838914]: received mask_obs with header seq: 287 +stamp: + secs: 1735525626 + nsecs: 840316772 +frame_id: '' and grasped_object_mask header seq: 66749 +stamp: + secs: 1735525626 + nsecs: 626195908 +frame_id: "camera_color_optical_frame" +[INFO] [1735525627.041523]: received mask_obs with header seq: 288 +stamp: + secs: 1735525627 + nsecs: 42562484 +frame_id: '' and grasped_object_mask header seq: 66755 +stamp: + secs: 1735525626 + nsecs: 826026440 +frame_id: "camera_color_optical_frame" +[INFO] [1735525627.238684]: received mask_obs with header seq: 289 +stamp: + secs: 1735525627 + nsecs: 240036964 +frame_id: '' and grasped_object_mask header seq: 66761 +stamp: + secs: 1735525627 + nsecs: 25856972 +frame_id: "camera_color_optical_frame" +[INFO] [1735525627.440479]: received mask_obs with header seq: 290 +stamp: + secs: 1735525627 + nsecs: 441817998 +frame_id: '' and grasped_object_mask header seq: 66767 +stamp: + secs: 1735525627 + nsecs: 225687265 +frame_id: "camera_color_optical_frame" +[INFO] [1735525627.637547]: received mask_obs with header seq: 291 +stamp: + secs: 1735525627 + nsecs: 638936042 +frame_id: '' and grasped_object_mask header seq: 66773 +stamp: + secs: 1735525627 + nsecs: 425517321 +frame_id: "camera_color_optical_frame" +[INFO] [1735525627.837747]: received mask_obs with header seq: 292 +stamp: + secs: 1735525627 + nsecs: 839163064 +frame_id: '' and grasped_object_mask header seq: 66779 +stamp: + secs: 1735525627 + nsecs: 625354290 +frame_id: "camera_color_optical_frame" +[INFO] [1735525628.038863]: received mask_obs with header seq: 293 +stamp: + secs: 1735525628 + nsecs: 40197849 +frame_id: '' and grasped_object_mask header seq: 66785 +stamp: + secs: 1735525627 + nsecs: 825191498 +frame_id: "camera_color_optical_frame" +[INFO] [1735525628.238286]: received mask_obs with header seq: 294 +stamp: + secs: 1735525628 + nsecs: 239512681 +frame_id: '' and grasped_object_mask header seq: 66791 +stamp: + secs: 1735525628 + nsecs: 25028467 +frame_id: "camera_color_optical_frame" +[INFO] [1735525628.439627]: received mask_obs with header seq: 295 +stamp: + secs: 1735525628 + nsecs: 441025257 +frame_id: '' and grasped_object_mask header seq: 66797 +stamp: + secs: 1735525628 + nsecs: 224865675 +frame_id: "camera_color_optical_frame" +[INFO] [1735525628.639546]: received mask_obs with header seq: 296 +stamp: + secs: 1735525628 + nsecs: 640854835 +frame_id: '' and grasped_object_mask header seq: 66803 +stamp: + secs: 1735525628 + nsecs: 424702644 +frame_id: "camera_color_optical_frame" +[INFO] [1735525628.841641]: received mask_obs with header seq: 297 +stamp: + secs: 1735525628 + nsecs: 842794656 +frame_id: '' and grasped_object_mask header seq: 66809 +stamp: + secs: 1735525628 + nsecs: 624541521 +frame_id: "camera_color_optical_frame" +[INFO] [1735525629.040022]: received mask_obs with header seq: 298 +stamp: + secs: 1735525629 + nsecs: 41417360 +frame_id: '' and grasped_object_mask header seq: 66815 +stamp: + secs: 1735525628 + nsecs: 824380875 +frame_id: "camera_color_optical_frame" +[INFO] [1735525629.238158]: received mask_obs with header seq: 299 +stamp: + secs: 1735525629 + nsecs: 239526748 +frame_id: '' and grasped_object_mask header seq: 66821 +stamp: + secs: 1735525629 + nsecs: 24220943 +frame_id: "camera_color_optical_frame" +[INFO] [1735525629.439627]: received mask_obs with header seq: 300 +stamp: + secs: 1735525629 + nsecs: 441032171 +frame_id: '' and grasped_object_mask header seq: 66827 +stamp: + secs: 1735525629 + nsecs: 224060297 +frame_id: "camera_color_optical_frame" +[INFO] [1735525629.638716]: received mask_obs with header seq: 301 +stamp: + secs: 1735525629 + nsecs: 640112638 +frame_id: '' and grasped_object_mask header seq: 66833 +stamp: + secs: 1735525629 + nsecs: 423899412 +frame_id: "camera_color_optical_frame" +[INFO] [1735525629.840999]: received mask_obs with header seq: 302 +stamp: + secs: 1735525629 + nsecs: 841911315 +frame_id: '' and grasped_object_mask header seq: 66839 +stamp: + secs: 1735525629 + nsecs: 623730659 +frame_id: "camera_color_optical_frame" +[INFO] [1735525630.042919]: received mask_obs with header seq: 303 +stamp: + secs: 1735525630 + nsecs: 42832612 +frame_id: '' and grasped_object_mask header seq: 66845 +stamp: + secs: 1735525629 + nsecs: 823559523 +frame_id: "camera_color_optical_frame" +[INFO] [1735525630.245613]: received mask_obs with header seq: 304 +stamp: + secs: 1735525630 + nsecs: 246883153 +frame_id: '' and grasped_object_mask header seq: 66851 +stamp: + secs: 1735525630 + nsecs: 23387909 +frame_id: "camera_color_optical_frame" +[INFO] [1735525630.437818]: received mask_obs with header seq: 305 +stamp: + secs: 1735525630 + nsecs: 439136505 +frame_id: '' and grasped_object_mask header seq: 66857 +stamp: + secs: 1735525630 + nsecs: 223216772 +frame_id: "camera_color_optical_frame" +[INFO] [1735525630.637493]: received mask_obs with header seq: 306 +stamp: + secs: 1735525630 + nsecs: 638891935 +frame_id: '' and grasped_object_mask header seq: 66863 +stamp: + secs: 1735525630 + nsecs: 423044443 +frame_id: "camera_color_optical_frame" +[INFO] [1735525630.837934]: received mask_obs with header seq: 307 +stamp: + secs: 1735525630 + nsecs: 839256048 +frame_id: '' and grasped_object_mask header seq: 66869 +stamp: + secs: 1735525630 + nsecs: 622881889 +frame_id: "camera_color_optical_frame" +[INFO] [1735525631.048399]: received mask_obs with header seq: 308 +stamp: + secs: 1735525631 + nsecs: 49698829 +frame_id: '' and grasped_object_mask header seq: 66875 +stamp: + secs: 1735525630 + nsecs: 822722197 +frame_id: "camera_color_optical_frame" +[INFO] [1735525631.237734]: received mask_obs with header seq: 309 +stamp: + secs: 1735525631 + nsecs: 239058017 +frame_id: '' and grasped_object_mask header seq: 66881 +stamp: + secs: 1735525631 + nsecs: 22562504 +frame_id: "camera_color_optical_frame" +[INFO] [1735525631.437447]: received mask_obs with header seq: 310 +stamp: + secs: 1735525631 + nsecs: 438802480 +frame_id: '' and grasped_object_mask header seq: 66887 +stamp: + secs: 1735525631 + nsecs: 222403049 +frame_id: "camera_color_optical_frame" +[INFO] [1735525631.636952]: received mask_obs with header seq: 311 +stamp: + secs: 1735525631 + nsecs: 638260126 +frame_id: '' and grasped_object_mask header seq: 66893 +stamp: + secs: 1735525631 + nsecs: 422244787 +frame_id: "camera_color_optical_frame" +[INFO] [1735525631.839637]: received mask_obs with header seq: 312 +stamp: + secs: 1735525631 + nsecs: 840984582 +frame_id: '' and grasped_object_mask header seq: 66899 +stamp: + secs: 1735525631 + nsecs: 622082472 +frame_id: "camera_color_optical_frame" +[INFO] [1735525632.038643]: received mask_obs with header seq: 313 +stamp: + secs: 1735525632 + nsecs: 40030479 +frame_id: '' and grasped_object_mask header seq: 66905 +stamp: + secs: 1735525631 + nsecs: 821919203 +frame_id: "camera_color_optical_frame" +[INFO] [1735525632.237243]: received mask_obs with header seq: 314 +stamp: + secs: 1735525632 + nsecs: 238662719 +frame_id: '' and grasped_object_mask header seq: 66911 +stamp: + secs: 1735525632 + nsecs: 21755695 +frame_id: "camera_color_optical_frame" +[INFO] [1735525632.445177]: received mask_obs with header seq: 315 +stamp: + secs: 1735525632 + nsecs: 446455001 +frame_id: '' and grasped_object_mask header seq: 66917 +stamp: + secs: 1735525632 + nsecs: 221591949 +frame_id: "camera_color_optical_frame" +[INFO] [1735525632.647617]: received mask_obs with header seq: 316 +stamp: + secs: 1735525632 + nsecs: 648725748 +frame_id: '' and grasped_object_mask header seq: 66923 +stamp: + secs: 1735525632 + nsecs: 421428204 +frame_id: "camera_color_optical_frame" +[INFO] [1735525632.841585]: received mask_obs with header seq: 317 +stamp: + secs: 1735525632 + nsecs: 841939687 +frame_id: '' and grasped_object_mask header seq: 66929 +stamp: + secs: 1735525632 + nsecs: 621270180 +frame_id: "camera_color_optical_frame" +[INFO] [1735525633.042540]: received mask_obs with header seq: 318 +stamp: + secs: 1735525633 + nsecs: 43486833 +frame_id: '' and grasped_object_mask header seq: 66935 +stamp: + secs: 1735525632 + nsecs: 821112871 +frame_id: "camera_color_optical_frame" +[INFO] [1735525633.237970]: received mask_obs with header seq: 319 +stamp: + secs: 1735525633 + nsecs: 239208459 +frame_id: '' and grasped_object_mask header seq: 66941 +stamp: + secs: 1735525633 + nsecs: 20955563 +frame_id: "camera_color_optical_frame" +[INFO] [1735525633.442179]: received mask_obs with header seq: 320 +stamp: + secs: 1735525633 + nsecs: 441583156 +frame_id: '' and grasped_object_mask header seq: 66947 +stamp: + secs: 1735525633 + nsecs: 220798254 +frame_id: "camera_color_optical_frame" +[INFO] [1735525633.639246]: received mask_obs with header seq: 321 +stamp: + secs: 1735525633 + nsecs: 640653133 +frame_id: '' and grasped_object_mask header seq: 66953 +stamp: + secs: 1735525633 + nsecs: 420641661 +frame_id: "camera_color_optical_frame" +[INFO] [1735525633.838366]: received mask_obs with header seq: 322 +stamp: + secs: 1735525633 + nsecs: 839763164 +frame_id: '' and grasped_object_mask header seq: 66959 +stamp: + secs: 1735525633 + nsecs: 620485306 +frame_id: "camera_color_optical_frame" +[INFO] [1735525634.040375]: received mask_obs with header seq: 323 +stamp: + secs: 1735525634 + nsecs: 40972471 +frame_id: '' and grasped_object_mask header seq: 66965 +stamp: + secs: 1735525633 + nsecs: 820330858 +frame_id: "camera_color_optical_frame" +[INFO] [1735525634.238549]: received mask_obs with header seq: 324 +stamp: + secs: 1735525634 + nsecs: 239665985 +frame_id: '' and grasped_object_mask header seq: 66971 +stamp: + secs: 1735525634 + nsecs: 20175457 +frame_id: "camera_color_optical_frame" +[INFO] [1735525634.438754]: received mask_obs with header seq: 325 +stamp: + secs: 1735525634 + nsecs: 439800262 +frame_id: '' and grasped_object_mask header seq: 66977 +stamp: + secs: 1735525634 + nsecs: 220020771 +frame_id: "camera_color_optical_frame" +[INFO] [1735525634.638125]: received mask_obs with header seq: 326 +stamp: + secs: 1735525634 + nsecs: 639502525 +frame_id: '' and grasped_object_mask header seq: 66983 +stamp: + secs: 1735525634 + nsecs: 419866323 +frame_id: "camera_color_optical_frame" +[INFO] [1735525634.838294]: received mask_obs with header seq: 327 +stamp: + secs: 1735525634 + nsecs: 839471578 +frame_id: '' and grasped_object_mask header seq: 66989 +stamp: + secs: 1735525634 + nsecs: 619710922 +frame_id: "camera_color_optical_frame" +[INFO] [1735525635.042547]: received mask_obs with header seq: 328 +stamp: + secs: 1735525635 + nsecs: 43111085 +frame_id: '' and grasped_object_mask header seq: 66995 +stamp: + secs: 1735525634 + nsecs: 819555521 +frame_id: "camera_color_optical_frame" +[INFO] [1735525635.238448]: received mask_obs with header seq: 329 +stamp: + secs: 1735525635 + nsecs: 239430665 +frame_id: '' and grasped_object_mask header seq: 67001 +stamp: + secs: 1735525635 + nsecs: 19401073 +frame_id: "camera_color_optical_frame" +[INFO] [1735525635.438004]: received mask_obs with header seq: 330 +stamp: + secs: 1735525635 + nsecs: 439445257 +frame_id: '' and grasped_object_mask header seq: 67007 +stamp: + secs: 1735525635 + nsecs: 219246387 +frame_id: "camera_color_optical_frame" +[INFO] [1735525635.641681]: received mask_obs with header seq: 331 +stamp: + secs: 1735525635 + nsecs: 643120527 +frame_id: '' and grasped_object_mask header seq: 67013 +stamp: + secs: 1735525635 + nsecs: 419091702 +frame_id: "camera_color_optical_frame" +[INFO] [1735525635.837395]: received mask_obs with header seq: 332 +stamp: + secs: 1735525635 + nsecs: 838777542 +frame_id: '' and grasped_object_mask header seq: 67019 +stamp: + secs: 1735525635 + nsecs: 618934870 +frame_id: "camera_color_optical_frame" +[INFO] [1735525636.041288]: received mask_obs with header seq: 333 +stamp: + secs: 1735525636 + nsecs: 41790008 +frame_id: '' and grasped_object_mask header seq: 67025 +stamp: + secs: 1735525635 + nsecs: 818777323 +frame_id: "camera_color_optical_frame" +[INFO] [1735525636.243478]: received mask_obs with header seq: 334 +stamp: + secs: 1735525636 + nsecs: 244686365 +frame_id: '' and grasped_object_mask header seq: 67031 +stamp: + secs: 1735525636 + nsecs: 18619776 +frame_id: "camera_color_optical_frame" +[INFO] [1735525636.445522]: received mask_obs with header seq: 335 +stamp: + secs: 1735525636 + nsecs: 446822643 +frame_id: '' and grasped_object_mask header seq: 67037 +stamp: + secs: 1735525636 + nsecs: 218462944 +frame_id: "camera_color_optical_frame" +[INFO] [1735525636.642196]: received mask_obs with header seq: 336 +stamp: + secs: 1735525636 + nsecs: 641846418 +frame_id: '' and grasped_object_mask header seq: 67043 +stamp: + secs: 1735525636 + nsecs: 418306828 +frame_id: "camera_color_optical_frame" +[INFO] [1735525636.846391]: received mask_obs with header seq: 337 +stamp: + secs: 1735525636 + nsecs: 847266912 +frame_id: '' and grasped_object_mask header seq: 67049 +stamp: + secs: 1735525636 + nsecs: 618144751 +frame_id: "camera_color_optical_frame" +[INFO] [1735525637.043741]: received mask_obs with header seq: 338 +stamp: + secs: 1735525637 + nsecs: 44598579 +frame_id: '' and grasped_object_mask header seq: 67055 +stamp: + secs: 1735525636 + nsecs: 817981005 +frame_id: "camera_color_optical_frame" +[INFO] [1735525637.238886]: received mask_obs with header seq: 339 +stamp: + secs: 1735525637 + nsecs: 239866733 +frame_id: '' and grasped_object_mask header seq: 67061 +stamp: + secs: 1735525637 + nsecs: 17817020 +frame_id: "camera_color_optical_frame" +[INFO] [1735525637.437526]: received mask_obs with header seq: 340 +stamp: + secs: 1735525637 + nsecs: 438924312 +frame_id: '' and grasped_object_mask header seq: 67067 +stamp: + secs: 1735525637 + nsecs: 217653036 +frame_id: "camera_color_optical_frame" +[INFO] [1735525637.636418]: received mask_obs with header seq: 341 +stamp: + secs: 1735525637 + nsecs: 637867450 +frame_id: '' and grasped_object_mask header seq: 67073 +stamp: + secs: 1735525637 + nsecs: 417489052 +frame_id: "camera_color_optical_frame" +[INFO] [1735525637.838046]: received mask_obs with header seq: 342 +stamp: + secs: 1735525637 + nsecs: 839435100 +frame_id: '' and grasped_object_mask header seq: 67079 +stamp: + secs: 1735525637 + nsecs: 617324591 +frame_id: "camera_color_optical_frame" +[INFO] [1735525638.040170]: received mask_obs with header seq: 343 +stamp: + secs: 1735525638 + nsecs: 40769338 +frame_id: '' and grasped_object_mask header seq: 67085 +stamp: + secs: 1735525637 + nsecs: 817158937 +frame_id: "camera_color_optical_frame" +[INFO] [1735525638.244294]: received mask_obs with header seq: 344 +stamp: + secs: 1735525638 + nsecs: 245126724 +frame_id: '' and grasped_object_mask header seq: 67091 +stamp: + secs: 1735525638 + nsecs: 16993523 +frame_id: "camera_color_optical_frame" +[INFO] [1735525638.440516]: received mask_obs with header seq: 345 +stamp: + secs: 1735525638 + nsecs: 440770626 +frame_id: '' and grasped_object_mask header seq: 67097 +stamp: + secs: 1735525638 + nsecs: 216827631 +frame_id: "camera_color_optical_frame" +[INFO] [1735525638.641548]: received mask_obs with header seq: 346 +stamp: + secs: 1735525638 + nsecs: 639351129 +frame_id: '' and grasped_object_mask header seq: 67103 +stamp: + secs: 1735525638 + nsecs: 416661978 +frame_id: "camera_color_optical_frame" +[INFO] [1735525638.836257]: received mask_obs with header seq: 347 +stamp: + secs: 1735525638 + nsecs: 837656736 +frame_id: '' and grasped_object_mask header seq: 67109 +stamp: + secs: 1735525638 + nsecs: 616493225 +frame_id: "camera_color_optical_frame" +[INFO] [1735525639.045340]: received mask_obs with header seq: 348 +stamp: + secs: 1735525639 + nsecs: 46572208 +frame_id: '' and grasped_object_mask header seq: 67115 +stamp: + secs: 1735525638 + nsecs: 816323996 +frame_id: "camera_color_optical_frame" +[INFO] [1735525639.236675]: received mask_obs with header seq: 349 +stamp: + secs: 1735525639 + nsecs: 238068580 +frame_id: '' and grasped_object_mask header seq: 67121 +stamp: + secs: 1735525639 + nsecs: 16153574 +frame_id: "camera_color_optical_frame" +[INFO] [1735525639.437499]: received mask_obs with header seq: 350 +stamp: + secs: 1735525639 + nsecs: 438865184 +frame_id: '' and grasped_object_mask header seq: 67127 +stamp: + secs: 1735525639 + nsecs: 215982437 +frame_id: "camera_color_optical_frame" +[INFO] [1735525639.636598]: received mask_obs with header seq: 351 +stamp: + secs: 1735525639 + nsecs: 637984991 +frame_id: '' and grasped_object_mask header seq: 67133 +stamp: + secs: 1735525639 + nsecs: 415811062 +frame_id: "camera_color_optical_frame" +[INFO] [1735525639.839956]: received mask_obs with header seq: 352 +stamp: + secs: 1735525639 + nsecs: 840347290 +frame_id: '' and grasped_object_mask header seq: 67139 +stamp: + secs: 1735525639 + nsecs: 615649223 +frame_id: "camera_color_optical_frame" +[INFO] [1735525640.040072]: received mask_obs with header seq: 353 +stamp: + secs: 1735525640 + nsecs: 40125131 +frame_id: '' and grasped_object_mask header seq: 67145 +stamp: + secs: 1735525639 + nsecs: 815490484 +frame_id: "camera_color_optical_frame" +[INFO] [1735525640.235307]: received mask_obs with header seq: 354 +stamp: + secs: 1735525640 + nsecs: 236719131 +frame_id: '' and grasped_object_mask header seq: 67151 +stamp: + secs: 1735525640 + nsecs: 15332937 +frame_id: "camera_color_optical_frame" +[INFO] [1735525640.439243]: received mask_obs with header seq: 355 +stamp: + secs: 1735525640 + nsecs: 439694881 +frame_id: '' and grasped_object_mask header seq: 67157 +stamp: + secs: 1735525640 + nsecs: 215174913 +frame_id: "camera_color_optical_frame" +[INFO] [1735525640.640588]: received mask_obs with header seq: 356 +stamp: + secs: 1735525640 + nsecs: 639163732 +frame_id: '' and grasped_object_mask header seq: 67163 +stamp: + secs: 1735525640 + nsecs: 415016651 +frame_id: "camera_color_optical_frame" +[INFO] [1735525640.837213]: received mask_obs with header seq: 357 +stamp: + secs: 1735525640 + nsecs: 838490486 +frame_id: '' and grasped_object_mask header seq: 67169 +stamp: + secs: 1735525640 + nsecs: 614855051 +frame_id: "camera_color_optical_frame" +[INFO] [1735525641.039195]: received mask_obs with header seq: 358 +stamp: + secs: 1735525641 + nsecs: 39801597 +frame_id: '' and grasped_object_mask header seq: 67175 +stamp: + secs: 1735525640 + nsecs: 814692020 +frame_id: "camera_color_optical_frame" +[INFO] [1735525641.237260]: received mask_obs with header seq: 359 +stamp: + secs: 1735525641 + nsecs: 238617181 +frame_id: '' and grasped_object_mask header seq: 67181 +stamp: + secs: 1735525641 + nsecs: 14528990 +frame_id: "camera_color_optical_frame" +[INFO] [1735525641.439611]: received mask_obs with header seq: 360 +stamp: + secs: 1735525641 + nsecs: 440734863 +frame_id: '' and grasped_object_mask header seq: 67187 +stamp: + secs: 1735525641 + nsecs: 214367151 +frame_id: "camera_color_optical_frame" +[INFO] [1735525641.636765]: received mask_obs with header seq: 361 +stamp: + secs: 1735525641 + nsecs: 638095855 +frame_id: '' and grasped_object_mask header seq: 67193 +stamp: + secs: 1735525641 + nsecs: 414203882 +frame_id: "camera_color_optical_frame" +[INFO] [1735525641.836219]: received mask_obs with header seq: 362 +stamp: + secs: 1735525641 + nsecs: 837707519 +frame_id: '' and grasped_object_mask header seq: 67199 +stamp: + secs: 1735525641 + nsecs: 614043951 +frame_id: "camera_color_optical_frame" +[INFO] [1735525642.042429]: received mask_obs with header seq: 363 +stamp: + secs: 1735525642 + nsecs: 43095350 +frame_id: '' and grasped_object_mask header seq: 67205 +stamp: + secs: 1735525641 + nsecs: 813884974 +frame_id: "camera_color_optical_frame" +[INFO] [1735525642.236079]: received mask_obs with header seq: 364 +stamp: + secs: 1735525642 + nsecs: 237419605 +frame_id: '' and grasped_object_mask header seq: 67211 +stamp: + secs: 1735525642 + nsecs: 13725996 +frame_id: "camera_color_optical_frame" +[INFO] [1735525642.437832]: received mask_obs with header seq: 365 +stamp: + secs: 1735525642 + nsecs: 438149690 +frame_id: '' and grasped_object_mask header seq: 67217 +stamp: + secs: 1735525642 + nsecs: 213567495 +frame_id: "camera_color_optical_frame" +[INFO] [1735525642.636402]: received mask_obs with header seq: 366 +stamp: + secs: 1735525642 + nsecs: 637671947 +frame_id: '' and grasped_object_mask header seq: 67223 +stamp: + secs: 1735525642 + nsecs: 413409948 +frame_id: "camera_color_optical_frame" +[INFO] [1735525642.838898]: received mask_obs with header seq: 367 +stamp: + secs: 1735525642 + nsecs: 838684320 +frame_id: '' and grasped_object_mask header seq: 67229 +stamp: + secs: 1735525642 + nsecs: 613247156 +frame_id: "camera_color_optical_frame" +[INFO] [1735525643.042407]: received mask_obs with header seq: 368 +stamp: + secs: 1735525643 + nsecs: 43395996 +frame_id: '' and grasped_object_mask header seq: 67235 +stamp: + secs: 1735525642 + nsecs: 813082457 +frame_id: "camera_color_optical_frame" +[INFO] [1735525643.242912]: received mask_obs with header seq: 369 +stamp: + secs: 1735525643 + nsecs: 244315624 +frame_id: '' and grasped_object_mask header seq: 67241 +stamp: + secs: 1735525643 + nsecs: 12917757 +frame_id: "camera_color_optical_frame" +[INFO] [1735525643.438872]: received mask_obs with header seq: 370 +stamp: + secs: 1735525643 + nsecs: 439556837 +frame_id: '' and grasped_object_mask header seq: 67247 +stamp: + secs: 1735525643 + nsecs: 212752581 +frame_id: "camera_color_optical_frame" +[INFO] [1735525643.637995]: received mask_obs with header seq: 371 +stamp: + secs: 1735525643 + nsecs: 638861894 +frame_id: '' and grasped_object_mask header seq: 67253 +stamp: + secs: 1735525643 + nsecs: 412587404 +frame_id: "camera_color_optical_frame" +[INFO] [1735525643.837169]: received mask_obs with header seq: 372 +stamp: + secs: 1735525643 + nsecs: 838426113 +frame_id: '' and grasped_object_mask header seq: 67259 +stamp: + secs: 1735525643 + nsecs: 612422943 +frame_id: "camera_color_optical_frame" +[INFO] [1735525644.039619]: received mask_obs with header seq: 373 +stamp: + secs: 1735525644 + nsecs: 39884805 +frame_id: '' and grasped_object_mask header seq: 67265 +stamp: + secs: 1735525643 + nsecs: 812259197 +frame_id: "camera_color_optical_frame" +[INFO] [1735525644.238214]: received mask_obs with header seq: 374 +stamp: + secs: 1735525644 + nsecs: 238584041 +frame_id: '' and grasped_object_mask header seq: 67271 +stamp: + secs: 1735525644 + nsecs: 12094736 +frame_id: "camera_color_optical_frame" +[INFO] [1735525644.442745]: received mask_obs with header seq: 375 +stamp: + secs: 1735525644 + nsecs: 444106578 +frame_id: '' and grasped_object_mask header seq: 67277 +stamp: + secs: 1735525644 + nsecs: 211930275 +frame_id: "camera_color_optical_frame" +[INFO] [1735525644.639258]: received mask_obs with header seq: 376 +stamp: + secs: 1735525644 + nsecs: 640053749 +frame_id: '' and grasped_object_mask header seq: 67283 +stamp: + secs: 1735525644 + nsecs: 411765337 +frame_id: "camera_color_optical_frame" +[INFO] [1735525644.836258]: received mask_obs with header seq: 377 +stamp: + secs: 1735525644 + nsecs: 837515115 +frame_id: '' and grasped_object_mask header seq: 67289 +stamp: + secs: 1735525644 + nsecs: 611606121 +frame_id: "camera_color_optical_frame" +[INFO] [1735525645.039170]: received mask_obs with header seq: 378 +stamp: + secs: 1735525645 + nsecs: 38934707 +frame_id: '' and grasped_object_mask header seq: 67295 +stamp: + secs: 1735525644 + nsecs: 811449051 +frame_id: "camera_color_optical_frame" +[INFO] [1735525645.238108]: received mask_obs with header seq: 379 +stamp: + secs: 1735525645 + nsecs: 238319873 +frame_id: '' and grasped_object_mask header seq: 67301 +stamp: + secs: 1735525645 + nsecs: 11292934 +frame_id: "camera_color_optical_frame" +[INFO] [1735525645.442921]: received mask_obs with header seq: 380 +stamp: + secs: 1735525645 + nsecs: 444331407 +frame_id: '' and grasped_object_mask header seq: 67307 +stamp: + secs: 1735525645 + nsecs: 211136580 +frame_id: "camera_color_optical_frame" +[INFO] [1735525645.636836]: received mask_obs with header seq: 381 +stamp: + secs: 1735525645 + nsecs: 637303590 +frame_id: '' and grasped_object_mask header seq: 67313 +stamp: + secs: 1735525645 + nsecs: 410979986 +frame_id: "camera_color_optical_frame" +[INFO] [1735525645.836623]: received mask_obs with header seq: 382 +stamp: + secs: 1735525645 + nsecs: 837404966 +frame_id: '' and grasped_object_mask header seq: 67319 +stamp: + secs: 1735525645 + nsecs: 610819578 +frame_id: "camera_color_optical_frame" +[INFO] [1735525646.037911]: received mask_obs with header seq: 383 +stamp: + secs: 1735525646 + nsecs: 38552761 +frame_id: '' and grasped_object_mask header seq: 67325 +stamp: + secs: 1735525645 + nsecs: 810657740 +frame_id: "camera_color_optical_frame" +[INFO] [1735525646.238733]: received mask_obs with header seq: 384 +stamp: + secs: 1735525646 + nsecs: 239892959 +frame_id: '' and grasped_object_mask header seq: 67331 +stamp: + secs: 1735525646 + nsecs: 10495901 +frame_id: "camera_color_optical_frame" +[INFO] [1735525646.431155]: received mask_obs with header seq: 385 +stamp: + secs: 1735525646 + nsecs: 432606458 +frame_id: '' and grasped_object_mask header seq: 67335 +stamp: + secs: 1735525646 + nsecs: 143722057 +frame_id: "camera_color_optical_frame" +[INFO] [1735525646.632610]: received mask_obs with header seq: 386 +stamp: + secs: 1735525646 + nsecs: 633690834 +frame_id: '' and grasped_object_mask header seq: 67343 +stamp: + secs: 1735525646 + nsecs: 410172939 +frame_id: "camera_color_optical_frame" +[INFO] [1735525646.833476]: received mask_obs with header seq: 387 +stamp: + secs: 1735525646 + nsecs: 834593772 +frame_id: '' and grasped_object_mask header seq: 67349 +stamp: + secs: 1735525646 + nsecs: 610008478 +frame_id: "camera_color_optical_frame" +[INFO] [1735525647.034635]: received mask_obs with header seq: 388 +stamp: + secs: 1735525647 + nsecs: 36032915 +frame_id: '' and grasped_object_mask header seq: 67355 +stamp: + secs: 1735525646 + nsecs: 809843063 +frame_id: "camera_color_optical_frame" +[INFO] [1735525647.242725]: received mask_obs with header seq: 389 +stamp: + secs: 1735525647 + nsecs: 243979454 +frame_id: '' and grasped_object_mask header seq: 67361 +stamp: + secs: 1735525647 + nsecs: 9677649 +frame_id: "camera_color_optical_frame" +[INFO] [1735525647.452163]: received mask_obs with header seq: 390 +stamp: + secs: 1735525647 + nsecs: 453361988 +frame_id: '' and grasped_object_mask header seq: 67367 +stamp: + secs: 1735525647 + nsecs: 209511995 +frame_id: "camera_color_optical_frame" +[INFO] [1735525647.649257]: received mask_obs with header seq: 391 +stamp: + secs: 1735525647 + nsecs: 650604724 +frame_id: '' and grasped_object_mask header seq: 67373 +stamp: + secs: 1735525647 + nsecs: 409347057 +frame_id: "camera_color_optical_frame" +[INFO] [1735525647.859020]: received mask_obs with header seq: 392 +stamp: + secs: 1735525647 + nsecs: 860123872 +frame_id: '' and grasped_object_mask header seq: 67379 +stamp: + secs: 1735525647 + nsecs: 609185696 +frame_id: "camera_color_optical_frame" +[INFO] [1735525648.059856]: received mask_obs with header seq: 393 +stamp: + secs: 1735525648 + nsecs: 58819770 +frame_id: '' and grasped_object_mask header seq: 67386 +stamp: + secs: 1735525647 + nsecs: 842333555 +frame_id: "camera_color_optical_frame" +[INFO] [1735525648.246214]: received mask_obs with header seq: 394 +stamp: + secs: 1735525648 + nsecs: 247459888 +frame_id: '' and grasped_object_mask header seq: 67391 +stamp: + secs: 1735525648 + nsecs: 8868456 +frame_id: "camera_color_optical_frame" +[INFO] [1735525648.440949]: received mask_obs with header seq: 395 +stamp: + secs: 1735525648 + nsecs: 442291259 +frame_id: '' and grasped_object_mask header seq: 67397 +stamp: + secs: 1735525648 + nsecs: 208709717 +frame_id: "camera_color_optical_frame" +[INFO] [1735525648.636853]: received mask_obs with header seq: 396 +stamp: + secs: 1735525648 + nsecs: 637920141 +frame_id: '' and grasped_object_mask header seq: 67403 +stamp: + secs: 1735525648 + nsecs: 408551216 +frame_id: "camera_color_optical_frame" +[INFO] [1735525648.833337]: received mask_obs with header seq: 397 +stamp: + secs: 1735525648 + nsecs: 833329439 +frame_id: '' and grasped_object_mask header seq: 67409 +stamp: + secs: 1735525648 + nsecs: 608392715 +frame_id: "camera_color_optical_frame" +[INFO] [1735525649.035335]: received mask_obs with header seq: 398 +stamp: + secs: 1735525649 + nsecs: 36109685 +frame_id: '' and grasped_object_mask header seq: 67415 +stamp: + secs: 1735525648 + nsecs: 808233023 +frame_id: "camera_color_optical_frame" +[INFO] [1735525649.078232]: Storing episode... +[print_text-8] killing on exit +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[INFO] [1735525649.235382]: received mask_obs with header seq: 399 +stamp: + secs: 1735525649 + nsecs: 236533403 +frame_id: '' and grasped_object_mask header seq: 67421 +stamp: + secs: 1735525649 + nsecs: 8073092 +frame_id: "camera_color_optical_frame" +[INFO] [1735525649.867484]: Stored episode 3. +[INFO] [1735525649.869428]: Saving video... +[INFO] [1735525650.320441]: Video saved! +Starting episode 3 +[INFO] [1735525650.365807]: resetting environment +[INFO] [1735525650.402042]: cleared current plan +[INFO] [1735525650.408072]: moving to home +[INFO] [1735525654.031784]: reached home +[INFO] [1735525654.036429]: reset action history +[INFO] [1735525656.888661]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735525665.021787]: resetting environment +[INFO] [1735525665.036329]: cleared current plan +[INFO] [1735525665.036519]: moving to home +[INFO] [1735525666.037243]: reached home +[INFO] [1735525666.041002]: reset action history +[INFO] [1735525668.827237]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/episode_rosbags/episode_3_2024-12-29-21-27-48.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1360290.log +started roslaunch server http://158.130.50.37:45567/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1361006] +process[print_text-11]: started with pid [1361007] +process[rosbag_record-12]: started with pid [1361008] +started bagging! +[INFO] [1735525669.833713]: received mask_obs with header seq: 400 +stamp: + secs: 1735525669 + nsecs: 834661722 +frame_id: '' and grasped_object_mask header seq: 68040 +stamp: + secs: 1735525669 + nsecs: 624771595 +frame_id: "camera_color_optical_frame" +[INFO] [1735525670.040195]: received mask_obs with header seq: 401 +stamp: + secs: 1735525670 + nsecs: 40144920 +frame_id: '' and grasped_object_mask header seq: 68046 +stamp: + secs: 1735525669 + nsecs: 824612617 +frame_id: "camera_color_optical_frame" +[INFO] [1735525670.234070]: received mask_obs with header seq: 402 +stamp: + secs: 1735525670 + nsecs: 235509157 +frame_id: '' and grasped_object_mask header seq: 68052 +stamp: + secs: 1735525670 + nsecs: 24453878 +frame_id: "camera_color_optical_frame" +[INFO] [1735525670.455362]: received mask_obs with header seq: 403 +stamp: + secs: 1735525670 + nsecs: 456707477 +frame_id: '' and grasped_object_mask header seq: 68058 +stamp: + secs: 1735525670 + nsecs: 224296093 +frame_id: "camera_color_optical_frame" +[INFO] [1735525670.641213]: received mask_obs with header seq: 404 +stamp: + secs: 1735525670 + nsecs: 642025232 +frame_id: '' and grasped_object_mask header seq: 68064 +stamp: + secs: 1735525670 + nsecs: 424137354 +frame_id: "camera_color_optical_frame" +[INFO] [1735525670.843978]: received mask_obs with header seq: 405 +stamp: + secs: 1735525670 + nsecs: 845165729 +frame_id: '' and grasped_object_mask header seq: 68070 +stamp: + secs: 1735525670 + nsecs: 623977900 +frame_id: "camera_color_optical_frame" +[INFO] [1735525671.050645]: received mask_obs with header seq: 406 +stamp: + secs: 1735525671 + nsecs: 50789594 +frame_id: '' and grasped_object_mask header seq: 68076 +stamp: + secs: 1735525670 + nsecs: 823817730 +frame_id: "camera_color_optical_frame" +[INFO] [1735525671.252804]: received mask_obs with header seq: 407 +stamp: + secs: 1735525671 + nsecs: 254127025 +frame_id: '' and grasped_object_mask header seq: 68082 +stamp: + secs: 1735525671 + nsecs: 23657560 +frame_id: "camera_color_optical_frame" +[INFO] [1735525671.452600]: received mask_obs with header seq: 408 +stamp: + secs: 1735525671 + nsecs: 453925132 +frame_id: '' and grasped_object_mask header seq: 68088 +stamp: + secs: 1735525671 + nsecs: 223497152 +frame_id: "camera_color_optical_frame" +[INFO] [1735525671.652801]: received mask_obs with header seq: 409 +stamp: + secs: 1735525671 + nsecs: 654187202 +frame_id: '' and grasped_object_mask header seq: 68094 +stamp: + secs: 1735525671 + nsecs: 423337936 +frame_id: "camera_color_optical_frame" +[INFO] [1735525671.852948]: received mask_obs with header seq: 410 +stamp: + secs: 1735525671 + nsecs: 854305267 +frame_id: '' and grasped_object_mask header seq: 68100 +stamp: + secs: 1735525671 + nsecs: 623173237 +frame_id: "camera_color_optical_frame" +[INFO] [1735525672.054452]: received mask_obs with header seq: 411 +stamp: + secs: 1735525672 + nsecs: 55764436 +frame_id: '' and grasped_object_mask header seq: 68106 +stamp: + secs: 1735525671 + nsecs: 823006868 +frame_id: "camera_color_optical_frame" +[INFO] [1735525672.257855]: received mask_obs with header seq: 412 +stamp: + secs: 1735525672 + nsecs: 259223461 +frame_id: '' and grasped_object_mask header seq: 68112 +stamp: + secs: 1735525672 + nsecs: 22839785 +frame_id: "camera_color_optical_frame" +[INFO] [1735525672.451590]: received mask_obs with header seq: 413 +stamp: + secs: 1735525672 + nsecs: 452951192 +frame_id: '' and grasped_object_mask header seq: 68118 +stamp: + secs: 1735525672 + nsecs: 222672939 +frame_id: "camera_color_optical_frame" +[INFO] [1735525672.661254]: received mask_obs with header seq: 414 +stamp: + secs: 1735525672 + nsecs: 662655591 +frame_id: '' and grasped_object_mask header seq: 68124 +stamp: + secs: 1735525672 + nsecs: 422505617 +frame_id: "camera_color_optical_frame" +[INFO] [1735525672.855889]: received mask_obs with header seq: 415 +stamp: + secs: 1735525672 + nsecs: 857256650 +frame_id: '' and grasped_object_mask header seq: 68130 +stamp: + secs: 1735525672 + nsecs: 622339725 +frame_id: "camera_color_optical_frame" +[INFO] [1735525673.054580]: received mask_obs with header seq: 416 +stamp: + secs: 1735525673 + nsecs: 55647373 +frame_id: '' and grasped_object_mask header seq: 68136 +stamp: + secs: 1735525672 + nsecs: 822175741 +frame_id: "camera_color_optical_frame" +[INFO] [1735525673.255053]: received mask_obs with header seq: 417 +stamp: + secs: 1735525673 + nsecs: 256139755 +frame_id: '' and grasped_object_mask header seq: 68142 +stamp: + secs: 1735525673 + nsecs: 22010565 +frame_id: "camera_color_optical_frame" +[INFO] [1735525673.453909]: received mask_obs with header seq: 418 +stamp: + secs: 1735525673 + nsecs: 455377101 +frame_id: '' and grasped_object_mask header seq: 68148 +stamp: + secs: 1735525673 + nsecs: 221845150 +frame_id: "camera_color_optical_frame" +[INFO] [1735525673.653644]: received mask_obs with header seq: 419 +stamp: + secs: 1735525673 + nsecs: 655071258 +frame_id: '' and grasped_object_mask header seq: 68154 +stamp: + secs: 1735525673 + nsecs: 421679497 +frame_id: "camera_color_optical_frame" +[INFO] [1735525673.852775]: received mask_obs with header seq: 420 +stamp: + secs: 1735525673 + nsecs: 854125976 +frame_id: '' and grasped_object_mask header seq: 68160 +stamp: + secs: 1735525673 + nsecs: 621514797 +frame_id: "camera_color_optical_frame" +[INFO] [1735525674.053858]: received mask_obs with header seq: 421 +stamp: + secs: 1735525674 + nsecs: 54855823 +frame_id: '' and grasped_object_mask header seq: 68166 +stamp: + secs: 1735525673 + nsecs: 821350574 +frame_id: "camera_color_optical_frame" +[INFO] [1735525674.260782]: received mask_obs with header seq: 422 +stamp: + secs: 1735525674 + nsecs: 262243270 +frame_id: '' and grasped_object_mask header seq: 68172 +stamp: + secs: 1735525674 + nsecs: 21187305 +frame_id: "camera_color_optical_frame" +[INFO] [1735525674.461262]: received mask_obs with header seq: 423 +stamp: + secs: 1735525674 + nsecs: 462690830 +frame_id: '' and grasped_object_mask header seq: 68178 +stamp: + secs: 1735525674 + nsecs: 221022606 +frame_id: "camera_color_optical_frame" +[INFO] [1735525674.653679]: received mask_obs with header seq: 424 +stamp: + secs: 1735525674 + nsecs: 654646873 +frame_id: '' and grasped_object_mask header seq: 68184 +stamp: + secs: 1735525674 + nsecs: 420858145 +frame_id: "camera_color_optical_frame" +[INFO] [1735525674.869609]: received mask_obs with header seq: 425 +stamp: + secs: 1735525674 + nsecs: 870880842 +frame_id: '' and grasped_object_mask header seq: 68190 +stamp: + secs: 1735525674 + nsecs: 620698214 +frame_id: "camera_color_optical_frame" +[INFO] [1735525675.070730]: received mask_obs with header seq: 426 +stamp: + secs: 1735525675 + nsecs: 71708917 +frame_id: '' and grasped_object_mask header seq: 68197 +stamp: + secs: 1735525674 + nsecs: 853848457 +frame_id: "camera_color_optical_frame" +[INFO] [1735525675.270386]: received mask_obs with header seq: 427 +stamp: + secs: 1735525675 + nsecs: 270505189 +frame_id: '' and grasped_object_mask header seq: 68203 +stamp: + secs: 1735525675 + nsecs: 53690910 +frame_id: "camera_color_optical_frame" +[INFO] [1735525675.469772]: received mask_obs with header seq: 428 +stamp: + secs: 1735525675 + nsecs: 471136808 +frame_id: '' and grasped_object_mask header seq: 68209 +stamp: + secs: 1735525675 + nsecs: 253533602 +frame_id: "camera_color_optical_frame" +[INFO] [1735525675.670278]: received mask_obs with header seq: 429 +stamp: + secs: 1735525675 + nsecs: 669900894 +frame_id: '' and grasped_object_mask header seq: 68215 +stamp: + secs: 1735525675 + nsecs: 453376770 +frame_id: "camera_color_optical_frame" +[INFO] [1735525675.866053]: received mask_obs with header seq: 430 +stamp: + secs: 1735525675 + nsecs: 867509841 +frame_id: '' and grasped_object_mask header seq: 68221 +stamp: + secs: 1735525675 + nsecs: 653213024 +frame_id: "camera_color_optical_frame" +[INFO] [1735525676.059744]: received mask_obs with header seq: 431 +stamp: + secs: 1735525676 + nsecs: 61170101 +frame_id: '' and grasped_object_mask header seq: 68227 +stamp: + secs: 1735525675 + nsecs: 853048801 +frame_id: "camera_color_optical_frame" +[INFO] [1735525676.252701]: received mask_obs with header seq: 432 +stamp: + secs: 1735525676 + nsecs: 254049301 +frame_id: '' and grasped_object_mask header seq: 68232 +stamp: + secs: 1735525676 + nsecs: 19577265 +frame_id: "camera_color_optical_frame" +[INFO] [1735525676.455055]: received mask_obs with header seq: 433 +stamp: + secs: 1735525676 + nsecs: 456290006 +frame_id: '' and grasped_object_mask header seq: 68238 +stamp: + secs: 1735525676 + nsecs: 219411373 +frame_id: "camera_color_optical_frame" +[INFO] [1735525676.662823]: received mask_obs with header seq: 434 +stamp: + secs: 1735525676 + nsecs: 664216518 +frame_id: '' and grasped_object_mask header seq: 68244 +stamp: + secs: 1735525676 + nsecs: 419246674 +frame_id: "camera_color_optical_frame" +[INFO] [1735525676.855555]: received mask_obs with header seq: 435 +stamp: + secs: 1735525676 + nsecs: 856695890 +frame_id: '' and grasped_object_mask header seq: 68250 +stamp: + secs: 1735525676 + nsecs: 619087458 +frame_id: "camera_color_optical_frame" +[INFO] [1735525677.054630]: received mask_obs with header seq: 436 +stamp: + secs: 1735525677 + nsecs: 56074142 +frame_id: '' and grasped_object_mask header seq: 68256 +stamp: + secs: 1735525676 + nsecs: 818931580 +frame_id: "camera_color_optical_frame" +[INFO] [1735525677.269140]: received mask_obs with header seq: 437 +stamp: + secs: 1735525677 + nsecs: 268072605 +frame_id: '' and grasped_object_mask header seq: 68262 +stamp: + secs: 1735525677 + nsecs: 18776655 +frame_id: "camera_color_optical_frame" +[INFO] [1735525677.459522]: received mask_obs with header seq: 438 +stamp: + secs: 1735525677 + nsecs: 460527420 +frame_id: '' and grasped_object_mask header seq: 68268 +stamp: + secs: 1735525677 + nsecs: 218621492 +frame_id: "camera_color_optical_frame" +[INFO] [1735525677.663296]: received mask_obs with header seq: 439 +stamp: + secs: 1735525677 + nsecs: 664698362 +frame_id: '' and grasped_object_mask header seq: 68275 +stamp: + secs: 1735525677 + nsecs: 451775312 +frame_id: "camera_color_optical_frame" +[INFO] [1735525677.869638]: received mask_obs with header seq: 440 +stamp: + secs: 1735525677 + nsecs: 868549823 +frame_id: '' and grasped_object_mask header seq: 68280 +stamp: + secs: 1735525677 + nsecs: 618311167 +frame_id: "camera_color_optical_frame" +[INFO] [1735525678.057452]: received mask_obs with header seq: 441 +stamp: + secs: 1735525678 + nsecs: 58795452 +frame_id: '' and grasped_object_mask header seq: 68286 +stamp: + secs: 1735525677 + nsecs: 818152905 +frame_id: "camera_color_optical_frame" +[INFO] [1735525678.258132]: received mask_obs with header seq: 442 +stamp: + secs: 1735525678 + nsecs: 259581089 +frame_id: '' and grasped_object_mask header seq: 68293 +stamp: + secs: 1735525678 + nsecs: 51301241 +frame_id: "camera_color_optical_frame" +[INFO] [1735525678.456389]: received mask_obs with header seq: 443 +stamp: + secs: 1735525678 + nsecs: 457814216 +frame_id: '' and grasped_object_mask header seq: 68298 +stamp: + secs: 1735525678 + nsecs: 217836857 +frame_id: "camera_color_optical_frame" +[INFO] [1735525678.657453]: received mask_obs with header seq: 444 +stamp: + secs: 1735525678 + nsecs: 658869504 +frame_id: '' and grasped_object_mask header seq: 68304 +stamp: + secs: 1735525678 + nsecs: 417679310 +frame_id: "camera_color_optical_frame" +[INFO] [1735525678.866705]: received mask_obs with header seq: 445 +stamp: + secs: 1735525678 + nsecs: 867809057 +frame_id: '' and grasped_object_mask header seq: 68310 +stamp: + secs: 1735525678 + nsecs: 617514849 +frame_id: "camera_color_optical_frame" +[INFO] [1735525679.060657]: received mask_obs with header seq: 446 +stamp: + secs: 1735525679 + nsecs: 61713457 +frame_id: '' and grasped_object_mask header seq: 68316 +stamp: + secs: 1735525678 + nsecs: 817348242 +frame_id: "camera_color_optical_frame" +[INFO] [1735525679.257376]: received mask_obs with header seq: 447 +stamp: + secs: 1735525679 + nsecs: 258741140 +frame_id: '' and grasped_object_mask header seq: 68322 +stamp: + secs: 1735525679 + nsecs: 17180204 +frame_id: "camera_color_optical_frame" +[INFO] [1735525679.455580]: received mask_obs with header seq: 448 +stamp: + secs: 1735525679 + nsecs: 456880569 +frame_id: '' and grasped_object_mask header seq: 68328 +stamp: + secs: 1735525679 + nsecs: 217011690 +frame_id: "camera_color_optical_frame" +[INFO] [1735525679.662668]: received mask_obs with header seq: 449 +stamp: + secs: 1735525679 + nsecs: 664032220 +frame_id: '' and grasped_object_mask header seq: 68334 +stamp: + secs: 1735525679 + nsecs: 416843176 +frame_id: "camera_color_optical_frame" +[INFO] [1735525679.868919]: received mask_obs with header seq: 450 +stamp: + secs: 1735525679 + nsecs: 869039058 +frame_id: '' and grasped_object_mask header seq: 68341 +stamp: + secs: 1735525679 + nsecs: 649985790 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 117708540 for nearest: 1735525679934225655 - target: 1735525679816517115 at idx 145 +[INFO] [1735525680.069995]: received mask_obs with header seq: 451 +stamp: + secs: 1735525680 + nsecs: 71392297 +frame_id: '' and grasped_object_mask header seq: 68346 +stamp: + secs: 1735525679 + nsecs: 816517115 +frame_id: "camera_color_optical_frame" +[INFO] [1735525680.262695]: received mask_obs with header seq: 452 +stamp: + secs: 1735525680 + nsecs: 264123916 +frame_id: '' and grasped_object_mask header seq: 68353 +stamp: + secs: 1735525680 + nsecs: 49661636 +frame_id: "camera_color_optical_frame" +[INFO] [1735525680.461733]: received mask_obs with header seq: 453 +stamp: + secs: 1735525680 + nsecs: 463148593 +frame_id: '' and grasped_object_mask header seq: 68358 +stamp: + secs: 1735525680 + nsecs: 216193676 +frame_id: "camera_color_optical_frame" +[INFO] [1735525680.662907]: received mask_obs with header seq: 454 +stamp: + secs: 1735525680 + nsecs: 664302349 +frame_id: '' and grasped_object_mask header seq: 68365 +stamp: + secs: 1735525680 + nsecs: 449337244 +frame_id: "camera_color_optical_frame" +[INFO] [1735525680.861674]: received mask_obs with header seq: 455 +stamp: + secs: 1735525680 + nsecs: 862854003 +frame_id: '' and grasped_object_mask header seq: 68371 +stamp: + secs: 1735525680 + nsecs: 649172068 +frame_id: "camera_color_optical_frame" +[INFO] [1735525681.062694]: received mask_obs with header seq: 456 +stamp: + secs: 1735525681 + nsecs: 64021110 +frame_id: '' and grasped_object_mask header seq: 68377 +stamp: + secs: 1735525680 + nsecs: 849005461 +frame_id: "camera_color_optical_frame" +[INFO] [1735525681.270610]: received mask_obs with header seq: 457 +stamp: + secs: 1735525681 + nsecs: 270998239 +frame_id: '' and grasped_object_mask header seq: 68383 +stamp: + secs: 1735525681 + nsecs: 48838377 +frame_id: "camera_color_optical_frame" +[INFO] [1735525681.459326]: received mask_obs with header seq: 458 +stamp: + secs: 1735525681 + nsecs: 460171461 +frame_id: '' and grasped_object_mask header seq: 68389 +stamp: + secs: 1735525681 + nsecs: 248670816 +frame_id: "camera_color_optical_frame" +[INFO] [1735525681.661413]: received mask_obs with header seq: 459 +stamp: + secs: 1735525681 + nsecs: 662475824 +frame_id: '' and grasped_object_mask header seq: 68395 +stamp: + secs: 1735525681 + nsecs: 448503494 +frame_id: "camera_color_optical_frame" +[INFO] [1735525681.868484]: received mask_obs with header seq: 460 +stamp: + secs: 1735525681 + nsecs: 869657516 +frame_id: '' and grasped_object_mask header seq: 68401 +stamp: + secs: 1735525681 + nsecs: 648341656 +frame_id: "camera_color_optical_frame" +[INFO] [1735525682.063498]: received mask_obs with header seq: 461 +stamp: + secs: 1735525682 + nsecs: 63333272 +frame_id: '' and grasped_object_mask header seq: 68407 +stamp: + secs: 1735525681 + nsecs: 848181248 +frame_id: "camera_color_optical_frame" +[INFO] [1735525682.259904]: received mask_obs with header seq: 462 +stamp: + secs: 1735525682 + nsecs: 261126995 +frame_id: '' and grasped_object_mask header seq: 68413 +stamp: + secs: 1735525682 + nsecs: 48021793 +frame_id: "camera_color_optical_frame" +[INFO] [1735525682.461337]: received mask_obs with header seq: 463 +stamp: + secs: 1735525682 + nsecs: 462590456 +frame_id: '' and grasped_object_mask header seq: 68419 +stamp: + secs: 1735525682 + nsecs: 247861385 +frame_id: "camera_color_optical_frame" +[INFO] [1735525682.667628]: received mask_obs with header seq: 464 +stamp: + secs: 1735525682 + nsecs: 668842792 +frame_id: '' and grasped_object_mask header seq: 68425 +stamp: + secs: 1735525682 + nsecs: 447701216 +frame_id: "camera_color_optical_frame" +[INFO] [1735525682.867312]: received mask_obs with header seq: 465 +stamp: + secs: 1735525682 + nsecs: 868556261 +frame_id: '' and grasped_object_mask header seq: 68431 +stamp: + secs: 1735525682 + nsecs: 647544622 +frame_id: "camera_color_optical_frame" +[INFO] [1735525683.068832]: received mask_obs with header seq: 466 +stamp: + secs: 1735525683 + nsecs: 70222616 +frame_id: '' and grasped_object_mask header seq: 68437 +stamp: + secs: 1735525682 + nsecs: 847389221 +frame_id: "camera_color_optical_frame" +[INFO] [1735525683.260877]: received mask_obs with header seq: 467 +stamp: + secs: 1735525683 + nsecs: 262154102 +frame_id: '' and grasped_object_mask header seq: 68443 +stamp: + secs: 1735525683 + nsecs: 47233820 +frame_id: "camera_color_optical_frame" +[INFO] [1735525683.459783]: received mask_obs with header seq: 468 +stamp: + secs: 1735525683 + nsecs: 460715293 +frame_id: '' and grasped_object_mask header seq: 68449 +stamp: + secs: 1735525683 + nsecs: 247079611 +frame_id: "camera_color_optical_frame" +[INFO] [1735525683.659568]: received mask_obs with header seq: 469 +stamp: + secs: 1735525683 + nsecs: 660382509 +frame_id: '' and grasped_object_mask header seq: 68455 +stamp: + secs: 1735525683 + nsecs: 446924686 +frame_id: "camera_color_optical_frame" +[INFO] [1735525683.862161]: received mask_obs with header seq: 470 +stamp: + secs: 1735525683 + nsecs: 861840963 +frame_id: '' and grasped_object_mask header seq: 68461 +stamp: + secs: 1735525683 + nsecs: 646763563 +frame_id: "camera_color_optical_frame" +[INFO] [1735525684.060914]: received mask_obs with header seq: 471 +stamp: + secs: 1735525684 + nsecs: 61739444 +frame_id: '' and grasped_object_mask header seq: 68467 +stamp: + secs: 1735525683 + nsecs: 846601009 +frame_id: "camera_color_optical_frame" +[INFO] [1735525684.256690]: received mask_obs with header seq: 472 +stamp: + secs: 1735525684 + nsecs: 258176565 +frame_id: '' and grasped_object_mask header seq: 68473 +stamp: + secs: 1735525684 + nsecs: 46438456 +frame_id: "camera_color_optical_frame" +[INFO] [1735525684.459600]: received mask_obs with header seq: 473 +stamp: + secs: 1735525684 + nsecs: 460678815 +frame_id: '' and grasped_object_mask header seq: 68479 +stamp: + secs: 1735525684 + nsecs: 246276140 +frame_id: "camera_color_optical_frame" +[INFO] [1735525684.662677]: received mask_obs with header seq: 474 +stamp: + secs: 1735525684 + nsecs: 662027597 +frame_id: '' and grasped_object_mask header seq: 68485 +stamp: + secs: 1735525684 + nsecs: 446114779 +frame_id: "camera_color_optical_frame" +[INFO] [1735525684.858857]: received mask_obs with header seq: 475 +stamp: + secs: 1735525684 + nsecs: 859993696 +frame_id: '' and grasped_object_mask header seq: 68491 +stamp: + secs: 1735525684 + nsecs: 645945072 +frame_id: "camera_color_optical_frame" +[INFO] [1735525685.061706]: received mask_obs with header seq: 476 +stamp: + secs: 1735525685 + nsecs: 61858415 +frame_id: '' and grasped_object_mask header seq: 68497 +stamp: + secs: 1735525684 + nsecs: 845773697 +frame_id: "camera_color_optical_frame" +[INFO] [1735525685.257368]: received mask_obs with header seq: 477 +stamp: + secs: 1735525685 + nsecs: 258623123 +frame_id: '' and grasped_object_mask header seq: 68503 +stamp: + secs: 1735525685 + nsecs: 45601845 +frame_id: "camera_color_optical_frame" +[INFO] [1735525685.457638]: received mask_obs with header seq: 478 +stamp: + secs: 1735525685 + nsecs: 458981513 +frame_id: '' and grasped_object_mask header seq: 68509 +stamp: + secs: 1735525685 + nsecs: 245429516 +frame_id: "camera_color_optical_frame" +[INFO] [1735525685.662191]: received mask_obs with header seq: 479 +stamp: + secs: 1735525685 + nsecs: 660420179 +frame_id: '' and grasped_object_mask header seq: 68515 +stamp: + secs: 1735525685 + nsecs: 445256948 +frame_id: "camera_color_optical_frame" +[INFO] [1735525685.857502]: received mask_obs with header seq: 480 +stamp: + secs: 1735525685 + nsecs: 858914375 +frame_id: '' and grasped_object_mask header seq: 68521 +stamp: + secs: 1735525685 + nsecs: 645093203 +frame_id: "camera_color_optical_frame" +[INFO] [1735525686.061204]: received mask_obs with header seq: 481 +stamp: + secs: 1735525686 + nsecs: 61766147 +frame_id: '' and grasped_object_mask header seq: 68527 +stamp: + secs: 1735525685 + nsecs: 844930410 +frame_id: "camera_color_optical_frame" +[INFO] [1735525686.262052]: received mask_obs with header seq: 482 +stamp: + secs: 1735525686 + nsecs: 260587692 +frame_id: '' and grasped_object_mask header seq: 68533 +stamp: + secs: 1735525686 + nsecs: 44767857 +frame_id: "camera_color_optical_frame" +[INFO] [1735525686.464152]: received mask_obs with header seq: 483 +stamp: + secs: 1735525686 + nsecs: 465391635 +frame_id: '' and grasped_object_mask header seq: 68539 +stamp: + secs: 1735525686 + nsecs: 244605064 +frame_id: "camera_color_optical_frame" +[INFO] [1735525686.657265]: received mask_obs with header seq: 484 +stamp: + secs: 1735525686 + nsecs: 658718347 +frame_id: '' and grasped_object_mask header seq: 68545 +stamp: + secs: 1735525686 + nsecs: 444442034 +frame_id: "camera_color_optical_frame" +[INFO] [1735525686.857791]: received mask_obs with header seq: 485 +stamp: + secs: 1735525686 + nsecs: 859148263 +frame_id: '' and grasped_object_mask header seq: 68551 +stamp: + secs: 1735525686 + nsecs: 644281864 +frame_id: "camera_color_optical_frame" +[INFO] [1735525687.062059]: received mask_obs with header seq: 486 +stamp: + secs: 1735525687 + nsecs: 62521934 +frame_id: '' and grasped_object_mask header seq: 68557 +stamp: + secs: 1735525686 + nsecs: 844123363 +frame_id: "camera_color_optical_frame" +[INFO] [1735525687.257706]: received mask_obs with header seq: 487 +stamp: + secs: 1735525687 + nsecs: 258896827 +frame_id: '' and grasped_object_mask header seq: 68563 +stamp: + secs: 1735525687 + nsecs: 43963909 +frame_id: "camera_color_optical_frame" +[INFO] [1735525687.458317]: received mask_obs with header seq: 488 +stamp: + secs: 1735525687 + nsecs: 459242343 +frame_id: '' and grasped_object_mask header seq: 68569 +stamp: + secs: 1735525687 + nsecs: 243804455 +frame_id: "camera_color_optical_frame" +[INFO] [1735525687.660781]: received mask_obs with header seq: 489 +stamp: + secs: 1735525687 + nsecs: 659337520 +frame_id: '' and grasped_object_mask header seq: 68575 +stamp: + secs: 1735525687 + nsecs: 443645239 +frame_id: "camera_color_optical_frame" +[INFO] [1735525687.863197]: received mask_obs with header seq: 490 +stamp: + secs: 1735525687 + nsecs: 864334821 +frame_id: '' and grasped_object_mask header seq: 68581 +stamp: + secs: 1735525687 + nsecs: 643486977 +frame_id: "camera_color_optical_frame" +[INFO] [1735525688.070674]: received mask_obs with header seq: 491 +stamp: + secs: 1735525688 + nsecs: 72070360 +frame_id: '' and grasped_object_mask header seq: 68587 +stamp: + secs: 1735525687 + nsecs: 843329191 +frame_id: "camera_color_optical_frame" +[INFO] [1735525688.266670]: received mask_obs with header seq: 492 +stamp: + secs: 1735525688 + nsecs: 267849683 +frame_id: '' and grasped_object_mask header seq: 68593 +stamp: + secs: 1735525688 + nsecs: 43171167 +frame_id: "camera_color_optical_frame" +[INFO] [1735525688.461686]: received mask_obs with header seq: 493 +stamp: + secs: 1735525688 + nsecs: 460575342 +frame_id: '' and grasped_object_mask header seq: 68599 +stamp: + secs: 1735525688 + nsecs: 243014336 +frame_id: "camera_color_optical_frame" +[INFO] [1735525688.659916]: received mask_obs with header seq: 494 +stamp: + secs: 1735525688 + nsecs: 659296512 +frame_id: '' and grasped_object_mask header seq: 68605 +stamp: + secs: 1735525688 + nsecs: 442856789 +frame_id: "camera_color_optical_frame" +[INFO] [1735525688.859200]: received mask_obs with header seq: 495 +stamp: + secs: 1735525688 + nsecs: 859605312 +frame_id: '' and grasped_object_mask header seq: 68611 +stamp: + secs: 1735525688 + nsecs: 642697573 +frame_id: "camera_color_optical_frame" +[INFO] [1735525689.063885]: received mask_obs with header seq: 496 +stamp: + secs: 1735525689 + nsecs: 65311908 +frame_id: '' and grasped_object_mask header seq: 68617 +stamp: + secs: 1735525688 + nsecs: 842538118 +frame_id: "camera_color_optical_frame" +[INFO] [1735525689.260669]: received mask_obs with header seq: 497 +stamp: + secs: 1735525689 + nsecs: 260173082 +frame_id: '' and grasped_object_mask header seq: 68623 +stamp: + secs: 1735525689 + nsecs: 42378664 +frame_id: "camera_color_optical_frame" +[INFO] [1735525689.463955]: received mask_obs with header seq: 498 +stamp: + secs: 1735525689 + nsecs: 464884996 +frame_id: '' and grasped_object_mask header seq: 68629 +stamp: + secs: 1735525689 + nsecs: 242219210 +frame_id: "camera_color_optical_frame" +[INFO] [1735525689.666533]: received mask_obs with header seq: 499 +stamp: + secs: 1735525689 + nsecs: 667950868 +frame_id: '' and grasped_object_mask header seq: 68635 +stamp: + secs: 1735525689 + nsecs: 442060947 +frame_id: "camera_color_optical_frame" +[INFO] [1735525689.859322]: received mask_obs with header seq: 500 +stamp: + secs: 1735525689 + nsecs: 859704017 +frame_id: '' and grasped_object_mask header seq: 68641 +stamp: + secs: 1735525689 + nsecs: 641900301 +frame_id: "camera_color_optical_frame" +[INFO] [1735525690.060483]: received mask_obs with header seq: 501 +stamp: + secs: 1735525690 + nsecs: 61123609 +frame_id: '' and grasped_object_mask header seq: 68647 +stamp: + secs: 1735525689 + nsecs: 841739416 +frame_id: "camera_color_optical_frame" +[INFO] [1735525690.256393]: received mask_obs with header seq: 502 +stamp: + secs: 1735525690 + nsecs: 257662296 +frame_id: '' and grasped_object_mask header seq: 68653 +stamp: + secs: 1735525690 + nsecs: 41578531 +frame_id: "camera_color_optical_frame" +[INFO] [1735525690.460047]: received mask_obs with header seq: 503 +stamp: + secs: 1735525690 + nsecs: 459004163 +frame_id: '' and grasped_object_mask header seq: 68659 +stamp: + secs: 1735525690 + nsecs: 241417408 +frame_id: "camera_color_optical_frame" +[INFO] [1735525690.656331]: received mask_obs with header seq: 504 +stamp: + secs: 1735525690 + nsecs: 657591819 +frame_id: '' and grasped_object_mask header seq: 68665 +stamp: + secs: 1735525690 + nsecs: 441256523 +frame_id: "camera_color_optical_frame" +[INFO] [1735525690.858937]: received mask_obs with header seq: 505 +stamp: + secs: 1735525690 + nsecs: 858067035 +frame_id: '' and grasped_object_mask header seq: 68671 +stamp: + secs: 1735525690 + nsecs: 641100883 +frame_id: "camera_color_optical_frame" +[INFO] [1735525691.064184]: received mask_obs with header seq: 506 +stamp: + secs: 1735525691 + nsecs: 65607070 +frame_id: '' and grasped_object_mask header seq: 68677 +stamp: + secs: 1735525690 + nsecs: 840945005 +frame_id: "camera_color_optical_frame" +[INFO] [1735525691.256818]: received mask_obs with header seq: 507 +stamp: + secs: 1735525691 + nsecs: 257932901 +frame_id: '' and grasped_object_mask header seq: 68683 +stamp: + secs: 1735525691 + nsecs: 40789843 +frame_id: "camera_color_optical_frame" +[INFO] [1735525691.456595]: received mask_obs with header seq: 508 +stamp: + secs: 1735525691 + nsecs: 457923889 +frame_id: '' and grasped_object_mask header seq: 68689 +stamp: + secs: 1735525691 + nsecs: 240634680 +frame_id: "camera_color_optical_frame" +[INFO] [1735525691.656454]: received mask_obs with header seq: 509 +stamp: + secs: 1735525691 + nsecs: 657652378 +frame_id: '' and grasped_object_mask header seq: 68695 +stamp: + secs: 1735525691 + nsecs: 440479994 +frame_id: "camera_color_optical_frame" +[INFO] [1735525691.856241]: received mask_obs with header seq: 510 +stamp: + secs: 1735525691 + nsecs: 857495307 +frame_id: '' and grasped_object_mask header seq: 68701 +stamp: + secs: 1735525691 + nsecs: 640318871 +frame_id: "camera_color_optical_frame" +[INFO] [1735525692.060078]: received mask_obs with header seq: 511 +stamp: + secs: 1735525692 + nsecs: 59800624 +frame_id: '' and grasped_object_mask header seq: 68707 +stamp: + secs: 1735525691 + nsecs: 840157032 +frame_id: "camera_color_optical_frame" +[INFO] [1735525692.258331]: received mask_obs with header seq: 512 +stamp: + secs: 1735525692 + nsecs: 258758068 +frame_id: '' and grasped_object_mask header seq: 68713 +stamp: + secs: 1735525692 + nsecs: 39994001 +frame_id: "camera_color_optical_frame" +[INFO] [1735525692.458251]: received mask_obs with header seq: 513 +stamp: + secs: 1735525692 + nsecs: 458121061 +frame_id: '' and grasped_object_mask header seq: 68719 +stamp: + secs: 1735525692 + nsecs: 239830971 +frame_id: "camera_color_optical_frame" +[INFO] [1735525692.655900]: received mask_obs with header seq: 514 +stamp: + secs: 1735525692 + nsecs: 657230615 +frame_id: '' and grasped_object_mask header seq: 68725 +stamp: + secs: 1735525692 + nsecs: 439667702 +frame_id: "camera_color_optical_frame" +[INFO] [1735525692.857930]: received mask_obs with header seq: 515 +stamp: + secs: 1735525692 + nsecs: 858763217 +frame_id: '' and grasped_object_mask header seq: 68731 +stamp: + secs: 1735525692 + nsecs: 639508247 +frame_id: "camera_color_optical_frame" +[INFO] [1735525693.060308]: received mask_obs with header seq: 516 +stamp: + secs: 1735525693 + nsecs: 60160875 +frame_id: '' and grasped_object_mask header seq: 68737 +stamp: + secs: 1735525692 + nsecs: 839349508 +frame_id: "camera_color_optical_frame" +[INFO] [1735525693.260556]: received mask_obs with header seq: 517 +stamp: + secs: 1735525693 + nsecs: 258938789 +frame_id: '' and grasped_object_mask header seq: 68743 +stamp: + secs: 1735525693 + nsecs: 39192200 +frame_id: "camera_color_optical_frame" +[INFO] [1735525693.458697]: received mask_obs with header seq: 518 +stamp: + secs: 1735525693 + nsecs: 457887649 +frame_id: '' and grasped_object_mask header seq: 68749 +stamp: + secs: 1735525693 + nsecs: 239033937 +frame_id: "camera_color_optical_frame" +[INFO] [1735525693.656423]: received mask_obs with header seq: 519 +stamp: + secs: 1735525693 + nsecs: 657163858 +frame_id: '' and grasped_object_mask header seq: 68755 +stamp: + secs: 1735525693 + nsecs: 438875914 +frame_id: "camera_color_optical_frame" +[INFO] [1735525693.859963]: received mask_obs with header seq: 520 +stamp: + secs: 1735525693 + nsecs: 859030723 +frame_id: '' and grasped_object_mask header seq: 68761 +stamp: + secs: 1735525693 + nsecs: 638713360 +frame_id: "camera_color_optical_frame" +[INFO] [1735525694.059111]: received mask_obs with header seq: 521 +stamp: + secs: 1735525694 + nsecs: 59247732 +frame_id: '' and grasped_object_mask header seq: 68767 +stamp: + secs: 1735525693 + nsecs: 838549614 +frame_id: "camera_color_optical_frame" +[INFO] [1735525694.255312]: received mask_obs with header seq: 522 +stamp: + secs: 1735525694 + nsecs: 256535530 +frame_id: '' and grasped_object_mask header seq: 68773 +stamp: + secs: 1735525694 + nsecs: 38385630 +frame_id: "camera_color_optical_frame" +[INFO] [1735525694.457225]: received mask_obs with header seq: 523 +stamp: + secs: 1735525694 + nsecs: 457334995 +frame_id: '' and grasped_object_mask header seq: 68779 +stamp: + secs: 1735525694 + nsecs: 238222837 +frame_id: "camera_color_optical_frame" +[INFO] [1735525694.655480]: received mask_obs with header seq: 524 +stamp: + secs: 1735525694 + nsecs: 656851530 +frame_id: '' and grasped_object_mask header seq: 68785 +stamp: + secs: 1735525694 + nsecs: 438059092 +frame_id: "camera_color_optical_frame" +[INFO] [1735525694.858520]: received mask_obs with header seq: 525 +stamp: + secs: 1735525694 + nsecs: 858904838 +frame_id: '' and grasped_object_mask header seq: 68791 +stamp: + secs: 1735525694 + nsecs: 637891054 +frame_id: "camera_color_optical_frame" +[INFO] [1735525695.061277]: received mask_obs with header seq: 526 +stamp: + secs: 1735525695 + nsecs: 62680006 +frame_id: '' and grasped_object_mask header seq: 68797 +stamp: + secs: 1735525694 + nsecs: 837721586 +frame_id: "camera_color_optical_frame" +[INFO] [1735525695.263050]: received mask_obs with header seq: 527 +stamp: + secs: 1735525695 + nsecs: 264299869 +frame_id: '' and grasped_object_mask header seq: 68803 +stamp: + secs: 1735525695 + nsecs: 37552118 +frame_id: "camera_color_optical_frame" +[INFO] [1735525695.458262]: received mask_obs with header seq: 528 +stamp: + secs: 1735525695 + nsecs: 458614587 +frame_id: '' and grasped_object_mask header seq: 68809 +stamp: + secs: 1735525695 + nsecs: 237381935 +frame_id: "camera_color_optical_frame" +[INFO] [1735525695.657817]: received mask_obs with header seq: 529 +stamp: + secs: 1735525695 + nsecs: 658830881 +frame_id: '' and grasped_object_mask header seq: 68815 +stamp: + secs: 1735525695 + nsecs: 437211275 +frame_id: "camera_color_optical_frame" +[INFO] [1735525695.857721]: received mask_obs with header seq: 530 +stamp: + secs: 1735525695 + nsecs: 857774496 +frame_id: '' and grasped_object_mask header seq: 68821 +stamp: + secs: 1735525695 + nsecs: 637050390 +frame_id: "camera_color_optical_frame" +[INFO] [1735525696.058403]: received mask_obs with header seq: 531 +stamp: + secs: 1735525696 + nsecs: 59771060 +frame_id: '' and grasped_object_mask header seq: 68827 +stamp: + secs: 1735525695 + nsecs: 836890936 +frame_id: "camera_color_optical_frame" +[INFO] [1735525696.105965]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1735525696.257850]: received mask_obs with header seq: 532 +stamp: + secs: 1735525696 + nsecs: 257751226 +frame_id: '' and grasped_object_mask header seq: 68833 +stamp: + secs: 1735525696 + nsecs: 36731720 +frame_id: "camera_color_optical_frame" +[INFO] [1735525696.891444]: Stored episode 4. +[INFO] [1735525696.894078]: Saving video... +[INFO] [1735525697.188771]: Video saved! +Starting episode 4 +[INFO] [1735525697.245848]: resetting environment +[INFO] [1735525697.265745]: cleared current plan +[INFO] [1735525697.265983]: moving to home +[INFO] [1735525700.876128]: reached home +[INFO] [1735525700.885114]: reset action history +[INFO] [1735525703.732547]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1735525713.134724]: resetting environment +[INFO] [1735525713.152972]: cleared current plan +[INFO] [1735525713.156857]: moving to home +[INFO] [1735525714.170815]: reached home +[INFO] [1735525714.176735]: reset action history +[INFO] [1735525717.008005]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/episode_rosbags/episode_4_2024-12-29-21-28-37.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1360290.log +started roslaunch server http://158.130.50.37:42573/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1361139] +process[print_text-14]: started with pid [1361140] +process[rosbag_record-15]: started with pid [1361141] +started bagging! +[INFO] [1735525718.097229]: received mask_obs with header seq: 533 +stamp: + secs: 1735525718 + nsecs: 97904682 +frame_id: '' and grasped_object_mask header seq: 69489 +stamp: + secs: 1735525717 + nsecs: 895236492 +frame_id: "camera_color_optical_frame" +[INFO] [1735525718.297083]: received mask_obs with header seq: 534 +stamp: + secs: 1735525718 + nsecs: 298511266 +frame_id: '' and grasped_object_mask header seq: 69495 +stamp: + secs: 1735525718 + nsecs: 94824314 +frame_id: "camera_color_optical_frame" +[INFO] [1735525718.498260]: received mask_obs with header seq: 535 +stamp: + secs: 1735525718 + nsecs: 499663829 +frame_id: '' and grasped_object_mask header seq: 69501 +stamp: + secs: 1735525718 + nsecs: 294399977 +frame_id: "camera_color_optical_frame" +[INFO] [1735525718.701608]: received mask_obs with header seq: 536 +stamp: + secs: 1735525718 + nsecs: 702931642 +frame_id: '' and grasped_object_mask header seq: 69507 +stamp: + secs: 1735525718 + nsecs: 493963957 +frame_id: "camera_color_optical_frame" +[INFO] [1735525718.907170]: received mask_obs with header seq: 537 +stamp: + secs: 1735525718 + nsecs: 908489942 +frame_id: '' and grasped_object_mask header seq: 69513 +stamp: + secs: 1735525718 + nsecs: 693561316 +frame_id: "camera_color_optical_frame" +[INFO] [1735525719.110836]: received mask_obs with header seq: 538 +stamp: + secs: 1735525719 + nsecs: 112069129 +frame_id: '' and grasped_object_mask header seq: 69519 +stamp: + secs: 1735525718 + nsecs: 893170118 +frame_id: "camera_color_optical_frame" +[INFO] [1735525719.313348]: received mask_obs with header seq: 539 +stamp: + secs: 1735525719 + nsecs: 314692020 +frame_id: '' and grasped_object_mask header seq: 69525 +stamp: + secs: 1735525719 + nsecs: 92768192 +frame_id: "camera_color_optical_frame" +[INFO] [1735525719.516299]: received mask_obs with header seq: 540 +stamp: + secs: 1735525719 + nsecs: 517522096 +frame_id: '' and grasped_object_mask header seq: 69531 +stamp: + secs: 1735525719 + nsecs: 292353868 +frame_id: "camera_color_optical_frame" +[INFO] [1735525719.717985]: received mask_obs with header seq: 541 +stamp: + secs: 1735525719 + nsecs: 719396591 +frame_id: '' and grasped_object_mask header seq: 69537 +stamp: + secs: 1735525719 + nsecs: 491927385 +frame_id: "camera_color_optical_frame" +[INFO] [1735525719.915575]: received mask_obs with header seq: 542 +stamp: + secs: 1735525719 + nsecs: 916935682 +frame_id: '' and grasped_object_mask header seq: 69543 +stamp: + secs: 1735525719 + nsecs: 691529751 +frame_id: "camera_color_optical_frame" +[INFO] [1735525720.116860]: received mask_obs with header seq: 543 +stamp: + secs: 1735525720 + nsecs: 118071317 +frame_id: '' and grasped_object_mask header seq: 69549 +stamp: + secs: 1735525719 + nsecs: 891140699 +frame_id: "camera_color_optical_frame" +[INFO] [1735525720.316134]: received mask_obs with header seq: 544 +stamp: + secs: 1735525720 + nsecs: 317301988 +frame_id: '' and grasped_object_mask header seq: 69555 +stamp: + secs: 1735525720 + nsecs: 90739727 +frame_id: "camera_color_optical_frame" +[INFO] [1735525720.514076]: received mask_obs with header seq: 545 +stamp: + secs: 1735525720 + nsecs: 515487909 +frame_id: '' and grasped_object_mask header seq: 69561 +stamp: + secs: 1735525720 + nsecs: 290328503 +frame_id: "camera_color_optical_frame" +[INFO] [1735525720.714904]: received mask_obs with header seq: 546 +stamp: + secs: 1735525720 + nsecs: 715960741 +frame_id: '' and grasped_object_mask header seq: 69567 +stamp: + secs: 1735525720 + nsecs: 489904881 +frame_id: "camera_color_optical_frame" +[INFO] [1735525720.914576]: received mask_obs with header seq: 547 +stamp: + secs: 1735525720 + nsecs: 915977001 +frame_id: '' and grasped_object_mask header seq: 69573 +stamp: + secs: 1735525720 + nsecs: 689507008 +frame_id: "camera_color_optical_frame" +[INFO] [1735525721.117723]: received mask_obs with header seq: 548 +stamp: + secs: 1735525721 + nsecs: 118780851 +frame_id: '' and grasped_object_mask header seq: 69579 +stamp: + secs: 1735525720 + nsecs: 889116526 +frame_id: "camera_color_optical_frame" +[INFO] [1735525721.317832]: received mask_obs with header seq: 549 +stamp: + secs: 1735525721 + nsecs: 319212436 +frame_id: '' and grasped_object_mask header seq: 69585 +stamp: + secs: 1735525721 + nsecs: 88714361 +frame_id: "camera_color_optical_frame" +[INFO] [1735525721.517066]: received mask_obs with header seq: 550 +stamp: + secs: 1735525721 + nsecs: 518519401 +frame_id: '' and grasped_object_mask header seq: 69591 +stamp: + secs: 1735525721 + nsecs: 288300514 +frame_id: "camera_color_optical_frame" +[INFO] [1735525721.717749]: received mask_obs with header seq: 551 +stamp: + secs: 1735525721 + nsecs: 719169378 +frame_id: '' and grasped_object_mask header seq: 69597 +stamp: + secs: 1735525721 + nsecs: 487875938 +frame_id: "camera_color_optical_frame" +[INFO] [1735525721.918758]: received mask_obs with header seq: 552 +stamp: + secs: 1735525721 + nsecs: 920112371 +frame_id: '' and grasped_object_mask header seq: 69603 +stamp: + secs: 1735525721 + nsecs: 687479496 +frame_id: "camera_color_optical_frame" +[INFO] [1735525722.118911]: received mask_obs with header seq: 553 +stamp: + secs: 1735525722 + nsecs: 120375156 +frame_id: '' and grasped_object_mask header seq: 69609 +stamp: + secs: 1735525721 + nsecs: 887091875 +frame_id: "camera_color_optical_frame" +[INFO] [1735525722.317340]: received mask_obs with header seq: 554 +stamp: + secs: 1735525722 + nsecs: 317537784 +frame_id: '' and grasped_object_mask header seq: 69615 +stamp: + secs: 1735525722 + nsecs: 86692572 +frame_id: "camera_color_optical_frame" +[INFO] [1735525722.520968]: received mask_obs with header seq: 555 +stamp: + secs: 1735525722 + nsecs: 522384405 +frame_id: '' and grasped_object_mask header seq: 69621 +stamp: + secs: 1735525722 + nsecs: 286282301 +frame_id: "camera_color_optical_frame" +[INFO] [1735525722.731386]: received mask_obs with header seq: 556 +stamp: + secs: 1735525722 + nsecs: 732505559 +frame_id: '' and grasped_object_mask header seq: 69627 +stamp: + secs: 1735525722 + nsecs: 485860348 +frame_id: "camera_color_optical_frame" +[INFO] [1735525722.922780]: received mask_obs with header seq: 557 +stamp: + secs: 1735525722 + nsecs: 923742294 +frame_id: '' and grasped_object_mask header seq: 69633 +stamp: + secs: 1735525722 + nsecs: 685465336 +frame_id: "camera_color_optical_frame" +[INFO] [1735525723.121860]: received mask_obs with header seq: 558 +stamp: + secs: 1735525723 + nsecs: 123044252 +frame_id: '' and grasped_object_mask header seq: 69639 +stamp: + secs: 1735525722 + nsecs: 885076761 +frame_id: "camera_color_optical_frame" +[INFO] [1735525723.320099]: received mask_obs with header seq: 559 +stamp: + secs: 1735525723 + nsecs: 321546792 +frame_id: '' and grasped_object_mask header seq: 69645 +stamp: + secs: 1735525723 + nsecs: 84676504 +frame_id: "camera_color_optical_frame" +[INFO] [1735525723.520699]: received mask_obs with header seq: 560 +stamp: + secs: 1735525723 + nsecs: 522161006 +frame_id: '' and grasped_object_mask header seq: 69651 +stamp: + secs: 1735525723 + nsecs: 284264803 +frame_id: "camera_color_optical_frame" +[INFO] [1735525723.719655]: received mask_obs with header seq: 561 +stamp: + secs: 1735525723 + nsecs: 721130132 +frame_id: '' and grasped_object_mask header seq: 69657 +stamp: + secs: 1735525723 + nsecs: 483841658 +frame_id: "camera_color_optical_frame" +[INFO] [1735525723.920523]: received mask_obs with header seq: 562 +stamp: + secs: 1735525723 + nsecs: 921966075 +frame_id: '' and grasped_object_mask header seq: 69663 +stamp: + secs: 1735525723 + nsecs: 683443308 +frame_id: "camera_color_optical_frame" +[INFO] [1735525724.116163]: received mask_obs with header seq: 563 +stamp: + secs: 1735525724 + nsecs: 117557525 +frame_id: '' and grasped_object_mask header seq: 69669 +stamp: + secs: 1735525723 + nsecs: 883052826 +frame_id: "camera_color_optical_frame" +[INFO] [1735525724.319823]: received mask_obs with header seq: 564 +stamp: + secs: 1735525724 + nsecs: 321152210 +frame_id: '' and grasped_object_mask header seq: 69675 +stamp: + secs: 1735525724 + nsecs: 82649469 +frame_id: "camera_color_optical_frame" +[INFO] [1735525724.528210]: received mask_obs with header seq: 565 +stamp: + secs: 1735525724 + nsecs: 529099225 +frame_id: '' and grasped_object_mask header seq: 69681 +stamp: + secs: 1735525724 + nsecs: 282234669 +frame_id: "camera_color_optical_frame" +[INFO] [1735525724.716816]: received mask_obs with header seq: 566 +stamp: + secs: 1735525724 + nsecs: 718209266 +frame_id: '' and grasped_object_mask header seq: 69687 +stamp: + secs: 1735525724 + nsecs: 481808186 +frame_id: "camera_color_optical_frame" +[INFO] [1735525724.925073]: received mask_obs with header seq: 567 +stamp: + secs: 1735525724 + nsecs: 926444530 +frame_id: '' and grasped_object_mask header seq: 69694 +stamp: + secs: 1735525724 + nsecs: 714679003 +frame_id: "camera_color_optical_frame" +[INFO] [1735525725.121503]: received mask_obs with header seq: 568 +stamp: + secs: 1735525725 + nsecs: 122957706 +frame_id: '' and grasped_object_mask header seq: 69699 +stamp: + secs: 1735525724 + nsecs: 881018877 +frame_id: "camera_color_optical_frame" +[INFO] [1735525725.319504]: received mask_obs with header seq: 569 +stamp: + secs: 1735525725 + nsecs: 320905923 +frame_id: '' and grasped_object_mask header seq: 69705 +stamp: + secs: 1735525725 + nsecs: 80617428 +frame_id: "camera_color_optical_frame" +[INFO] [1735525725.518985]: received mask_obs with header seq: 570 +stamp: + secs: 1735525725 + nsecs: 519962072 +frame_id: '' and grasped_object_mask header seq: 69712 +stamp: + secs: 1735525725 + nsecs: 313466311 +frame_id: "camera_color_optical_frame" +[INFO] [1735525725.721748]: received mask_obs with header seq: 571 +stamp: + secs: 1735525725 + nsecs: 723153352 +frame_id: '' and grasped_object_mask header seq: 69718 +stamp: + secs: 1735525725 + nsecs: 513038874 +frame_id: "camera_color_optical_frame" +[INFO] [1735525725.922148]: received mask_obs with header seq: 572 +stamp: + secs: 1735525725 + nsecs: 923576354 +frame_id: '' and grasped_object_mask header seq: 69724 +stamp: + secs: 1735525725 + nsecs: 712640524 +frame_id: "camera_color_optical_frame" +[INFO] [1735525726.123593]: received mask_obs with header seq: 573 +stamp: + secs: 1735525726 + nsecs: 124872446 +frame_id: '' and grasped_object_mask header seq: 69730 +stamp: + secs: 1735525725 + nsecs: 912237644 +frame_id: "camera_color_optical_frame" +[INFO] [1735525726.322240]: received mask_obs with header seq: 574 +stamp: + secs: 1735525726 + nsecs: 323561668 +frame_id: '' and grasped_object_mask header seq: 69736 +stamp: + secs: 1735525726 + nsecs: 111822605 +frame_id: "camera_color_optical_frame" +[INFO] [1735525726.522965]: received mask_obs with header seq: 575 +stamp: + secs: 1735525726 + nsecs: 524131298 +frame_id: '' and grasped_object_mask header seq: 69741 +stamp: + secs: 1735525726 + nsecs: 278133392 +frame_id: "camera_color_optical_frame" +[INFO] [1735525726.721524]: received mask_obs with header seq: 576 +stamp: + secs: 1735525726 + nsecs: 722773313 +frame_id: '' and grasped_object_mask header seq: 69748 +stamp: + secs: 1735525726 + nsecs: 510955811 +frame_id: "camera_color_optical_frame" +[INFO] [1735525726.924208]: received mask_obs with header seq: 577 +stamp: + secs: 1735525726 + nsecs: 925590276 +frame_id: '' and grasped_object_mask header seq: 69754 +stamp: + secs: 1735525726 + nsecs: 710560322 +frame_id: "camera_color_optical_frame" +[INFO] [1735525727.124770]: received mask_obs with header seq: 578 +stamp: + secs: 1735525727 + nsecs: 125960588 +frame_id: '' and grasped_object_mask header seq: 69760 +stamp: + secs: 1735525726 + nsecs: 910165548 +frame_id: "camera_color_optical_frame" +[INFO] [1735525727.321727]: received mask_obs with header seq: 579 +stamp: + secs: 1735525727 + nsecs: 323161602 +frame_id: '' and grasped_object_mask header seq: 69766 +stamp: + secs: 1735525727 + nsecs: 109757900 +frame_id: "camera_color_optical_frame" +[INFO] [1735525727.523484]: received mask_obs with header seq: 580 +stamp: + secs: 1735525727 + nsecs: 524909257 +frame_id: '' and grasped_object_mask header seq: 69772 +stamp: + secs: 1735525727 + nsecs: 309338570 +frame_id: "camera_color_optical_frame" +[INFO] [1735525727.724627]: received mask_obs with header seq: 581 +stamp: + secs: 1735525727 + nsecs: 725836753 +frame_id: '' and grasped_object_mask header seq: 69778 +stamp: + secs: 1735525727 + nsecs: 508907557 +frame_id: "camera_color_optical_frame" +[INFO] [1735525727.928914]: received mask_obs with header seq: 582 +stamp: + secs: 1735525727 + nsecs: 929182767 +frame_id: '' and grasped_object_mask header seq: 69784 +stamp: + secs: 1735525727 + nsecs: 708507776 +frame_id: "camera_color_optical_frame" +[INFO] [1735525728.131729]: received mask_obs with header seq: 583 +stamp: + secs: 1735525728 + nsecs: 132394075 +frame_id: '' and grasped_object_mask header seq: 69790 +stamp: + secs: 1735525727 + nsecs: 908104420 +frame_id: "camera_color_optical_frame" +[INFO] [1735525728.334437]: received mask_obs with header seq: 584 +stamp: + secs: 1735525728 + nsecs: 335874319 +frame_id: '' and grasped_object_mask header seq: 69796 +stamp: + secs: 1735525728 + nsecs: 107689142 +frame_id: "camera_color_optical_frame" +[INFO] [1735525728.530137]: received mask_obs with header seq: 585 +stamp: + secs: 1735525728 + nsecs: 531518459 +frame_id: '' and grasped_object_mask header seq: 69802 +stamp: + secs: 1735525728 + nsecs: 307262421 +frame_id: "camera_color_optical_frame" +[INFO] [1735525728.722549]: received mask_obs with header seq: 586 +stamp: + secs: 1735525728 + nsecs: 723639726 +frame_id: '' and grasped_object_mask header seq: 69808 +stamp: + secs: 1735525728 + nsecs: 506822824 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -240067020 for nearest: 1735525728467530236 - target: 1735525728707597256 at idx 163 +[INFO] [1735525728.923027]: received mask_obs with header seq: 587 +stamp: + secs: 1735525728 + nsecs: 924468994 +frame_id: '' and grasped_object_mask header seq: 69814 +stamp: + secs: 1735525728 + nsecs: 707597256 +frame_id: "camera_color_optical_frame" +[INFO] [1735525729.122415]: received mask_obs with header seq: 588 +stamp: + secs: 1735525729 + nsecs: 123872995 +frame_id: '' and grasped_object_mask header seq: 69820 +stamp: + secs: 1735525728 + nsecs: 909572840 +frame_id: "camera_color_optical_frame" +[INFO] [1735525729.324274]: received mask_obs with header seq: 589 +stamp: + secs: 1735525729 + nsecs: 325144052 +frame_id: '' and grasped_object_mask header seq: 69826 +stamp: + secs: 1735525729 + nsecs: 111624718 +frame_id: "camera_color_optical_frame" +[INFO] [1735525729.521239]: received mask_obs with header seq: 590 +stamp: + secs: 1735525729 + nsecs: 522616863 +frame_id: '' and grasped_object_mask header seq: 69832 +stamp: + secs: 1735525729 + nsecs: 313752890 +frame_id: "camera_color_optical_frame" +[INFO] [1735525729.721219]: received mask_obs with header seq: 591 +stamp: + secs: 1735525729 + nsecs: 722599267 +frame_id: '' and grasped_object_mask header seq: 69838 +stamp: + secs: 1735525729 + nsecs: 515958309 +frame_id: "camera_color_optical_frame" +[INFO] [1735525729.921437]: received mask_obs with header seq: 592 +stamp: + secs: 1735525729 + nsecs: 922796487 +frame_id: '' and grasped_object_mask header seq: 69844 +stamp: + secs: 1735525729 + nsecs: 716793299 +frame_id: "camera_color_optical_frame" +[INFO] [1735525730.121734]: received mask_obs with header seq: 593 +stamp: + secs: 1735525730 + nsecs: 123070240 +frame_id: '' and grasped_object_mask header seq: 69850 +stamp: + secs: 1735525729 + nsecs: 916184664 +frame_id: "camera_color_optical_frame" +[INFO] [1735525730.321382]: received mask_obs with header seq: 594 +stamp: + secs: 1735525730 + nsecs: 322851657 +frame_id: '' and grasped_object_mask header seq: 69856 +stamp: + secs: 1735525730 + nsecs: 115553141 +frame_id: "camera_color_optical_frame" +[INFO] [1735525730.522092]: received mask_obs with header seq: 595 +stamp: + secs: 1735525730 + nsecs: 523485183 +frame_id: '' and grasped_object_mask header seq: 69862 +stamp: + secs: 1735525730 + nsecs: 314899206 +frame_id: "camera_color_optical_frame" +[INFO] [1735525730.720519]: received mask_obs with header seq: 596 +stamp: + secs: 1735525730 + nsecs: 721915721 +frame_id: '' and grasped_object_mask header seq: 69868 +stamp: + secs: 1735525730 + nsecs: 514222383 +frame_id: "camera_color_optical_frame" +[INFO] [1735525730.923713]: received mask_obs with header seq: 597 +stamp: + secs: 1735525730 + nsecs: 924848079 +frame_id: '' and grasped_object_mask header seq: 69874 +stamp: + secs: 1735525730 + nsecs: 714205027 +frame_id: "camera_color_optical_frame" +[INFO] [1735525731.120961]: received mask_obs with header seq: 598 +stamp: + secs: 1735525731 + nsecs: 122398614 +frame_id: '' and grasped_object_mask header seq: 69880 +stamp: + secs: 1735525730 + nsecs: 914919138 +frame_id: "camera_color_optical_frame" +[INFO] [1735525731.325046]: received mask_obs with header seq: 599 +stamp: + secs: 1735525731 + nsecs: 323412418 +frame_id: '' and grasped_object_mask header seq: 69886 +stamp: + secs: 1735525731 + nsecs: 115709066 +frame_id: "camera_color_optical_frame" +[INFO] [1735525731.528188]: received mask_obs with header seq: 600 +stamp: + secs: 1735525731 + nsecs: 529625654 +frame_id: '' and grasped_object_mask header seq: 69892 +stamp: + secs: 1735525731 + nsecs: 316574335 +frame_id: "camera_color_optical_frame" +[INFO] [1735525731.721414]: received mask_obs with header seq: 601 +stamp: + secs: 1735525731 + nsecs: 722804069 +frame_id: '' and grasped_object_mask header seq: 69898 +stamp: + secs: 1735525731 + nsecs: 517514944 +frame_id: "camera_color_optical_frame" +[INFO] [1735525731.922000]: received mask_obs with header seq: 602 +stamp: + secs: 1735525731 + nsecs: 923108339 +frame_id: '' and grasped_object_mask header seq: 69904 +stamp: + secs: 1735525731 + nsecs: 717822790 +frame_id: "camera_color_optical_frame" +[INFO] [1735525732.129355]: received mask_obs with header seq: 603 +stamp: + secs: 1735525732 + nsecs: 130804777 +frame_id: '' and grasped_object_mask header seq: 69910 +stamp: + secs: 1735525731 + nsecs: 917433262 +frame_id: "camera_color_optical_frame" +[INFO] [1735525732.320113]: received mask_obs with header seq: 604 +stamp: + secs: 1735525732 + nsecs: 321427822 +frame_id: '' and grasped_object_mask header seq: 69916 +stamp: + secs: 1735525732 + nsecs: 117031813 +frame_id: "camera_color_optical_frame" +[INFO] [1735525732.523925]: received mask_obs with header seq: 605 +stamp: + secs: 1735525732 + nsecs: 523463010 +frame_id: '' and grasped_object_mask header seq: 69922 +stamp: + secs: 1735525732 + nsecs: 316618443 +frame_id: "camera_color_optical_frame" +[INFO] [1735525732.719994]: received mask_obs with header seq: 606 +stamp: + secs: 1735525732 + nsecs: 721429347 +frame_id: '' and grasped_object_mask header seq: 69928 +stamp: + secs: 1735525732 + nsecs: 516193628 +frame_id: "camera_color_optical_frame" +[INFO] [1735525732.919884]: received mask_obs with header seq: 607 +stamp: + secs: 1735525732 + nsecs: 921360015 +frame_id: '' and grasped_object_mask header seq: 69934 +stamp: + secs: 1735525732 + nsecs: 715780735 +frame_id: "camera_color_optical_frame" +[INFO] [1735525733.120186]: received mask_obs with header seq: 608 +stamp: + secs: 1735525733 + nsecs: 121605873 +frame_id: '' and grasped_object_mask header seq: 69940 +stamp: + secs: 1735525732 + nsecs: 915379524 +frame_id: "camera_color_optical_frame" +[INFO] [1735525733.319634]: received mask_obs with header seq: 609 +stamp: + secs: 1735525733 + nsecs: 320922613 +frame_id: '' and grasped_object_mask header seq: 69946 +stamp: + secs: 1735525733 + nsecs: 114967108 +frame_id: "camera_color_optical_frame" +[INFO] [1735525733.523229]: received mask_obs with header seq: 610 +stamp: + secs: 1735525733 + nsecs: 524336338 +frame_id: '' and grasped_object_mask header seq: 69952 +stamp: + secs: 1735525733 + nsecs: 314541340 +frame_id: "camera_color_optical_frame" +[INFO] [1735525733.719064]: received mask_obs with header seq: 611 +stamp: + secs: 1735525733 + nsecs: 720572710 +frame_id: '' and grasped_object_mask header seq: 69958 +stamp: + secs: 1735525733 + nsecs: 514103413 +frame_id: "camera_color_optical_frame" +[INFO] [1735525733.921723]: received mask_obs with header seq: 612 +stamp: + secs: 1735525733 + nsecs: 921228885 +frame_id: '' and grasped_object_mask header seq: 69964 +stamp: + secs: 1735525733 + nsecs: 713684559 +frame_id: "camera_color_optical_frame" +[INFO] [1735525734.122946]: received mask_obs with header seq: 613 +stamp: + secs: 1735525734 + nsecs: 122863769 +frame_id: '' and grasped_object_mask header seq: 69970 +stamp: + secs: 1735525733 + nsecs: 913284779 +frame_id: "camera_color_optical_frame" +[INFO] [1735525734.321328]: received mask_obs with header seq: 614 +stamp: + secs: 1735525734 + nsecs: 322721719 +frame_id: '' and grasped_object_mask header seq: 69976 +stamp: + secs: 1735525734 + nsecs: 112872362 +frame_id: "camera_color_optical_frame" +[INFO] [1735525734.526064]: received mask_obs with header seq: 615 +stamp: + secs: 1735525734 + nsecs: 527472972 +frame_id: '' and grasped_object_mask header seq: 69982 +stamp: + secs: 1735525734 + nsecs: 312448263 +frame_id: "camera_color_optical_frame" +[INFO] [1735525734.721317]: received mask_obs with header seq: 616 +stamp: + secs: 1735525734 + nsecs: 720963239 +frame_id: '' and grasped_object_mask header seq: 69988 +stamp: + secs: 1735525734 + nsecs: 512012720 +frame_id: "camera_color_optical_frame" +[INFO] [1735525734.918957]: received mask_obs with header seq: 617 +stamp: + secs: 1735525734 + nsecs: 920322418 +frame_id: '' and grasped_object_mask header seq: 69994 +stamp: + secs: 1735525734 + nsecs: 711599827 +frame_id: "camera_color_optical_frame" +[INFO] [1735525735.120779]: received mask_obs with header seq: 618 +stamp: + secs: 1735525735 + nsecs: 121833562 +frame_id: '' and grasped_object_mask header seq: 70000 +stamp: + secs: 1735525734 + nsecs: 911211252 +frame_id: "camera_color_optical_frame" +[INFO] [1735525735.317885]: received mask_obs with header seq: 619 +stamp: + secs: 1735525735 + nsecs: 319349050 +frame_id: '' and grasped_object_mask header seq: 70006 +stamp: + secs: 1735525735 + nsecs: 110810757 +frame_id: "camera_color_optical_frame" +[INFO] [1735525735.521458]: received mask_obs with header seq: 620 +stamp: + secs: 1735525735 + nsecs: 520875453 +frame_id: '' and grasped_object_mask header seq: 70012 +stamp: + secs: 1735525735 + nsecs: 310398817 +frame_id: "camera_color_optical_frame" +[INFO] [1735525735.726978]: received mask_obs with header seq: 621 +stamp: + secs: 1735525735 + nsecs: 728170394 +frame_id: '' and grasped_object_mask header seq: 70018 +stamp: + secs: 1735525735 + nsecs: 509975433 +frame_id: "camera_color_optical_frame" +[INFO] [1735525735.920434]: received mask_obs with header seq: 622 +stamp: + secs: 1735525735 + nsecs: 921442985 +frame_id: '' and grasped_object_mask header seq: 70024 +stamp: + secs: 1735525735 + nsecs: 709568739 +frame_id: "camera_color_optical_frame" +[INFO] [1735525736.122182]: received mask_obs with header seq: 623 +stamp: + secs: 1735525736 + nsecs: 123475790 +frame_id: '' and grasped_object_mask header seq: 70030 +stamp: + secs: 1735525735 + nsecs: 909177065 +frame_id: "camera_color_optical_frame" +[INFO] [1735525736.320038]: received mask_obs with header seq: 624 +stamp: + secs: 1735525736 + nsecs: 321369409 +frame_id: '' and grasped_object_mask header seq: 70036 +stamp: + secs: 1735525736 + nsecs: 108773708 +frame_id: "camera_color_optical_frame" +[INFO] [1735525736.520740]: received mask_obs with header seq: 625 +stamp: + secs: 1735525736 + nsecs: 521342992 +frame_id: '' and grasped_object_mask header seq: 70042 +stamp: + secs: 1735525736 + nsecs: 308358669 +frame_id: "camera_color_optical_frame" +[INFO] [1735525736.720304]: received mask_obs with header seq: 626 +stamp: + secs: 1735525736 + nsecs: 720999717 +frame_id: '' and grasped_object_mask header seq: 70048 +stamp: + secs: 1735525736 + nsecs: 507931948 +frame_id: "camera_color_optical_frame" +[INFO] [1735525736.920063]: received mask_obs with header seq: 627 +stamp: + secs: 1735525736 + nsecs: 920675754 +frame_id: '' and grasped_object_mask header seq: 70054 +stamp: + secs: 1735525736 + nsecs: 707522392 +frame_id: "camera_color_optical_frame" +[INFO] [1735525737.120909]: received mask_obs with header seq: 628 +stamp: + secs: 1735525737 + nsecs: 120794057 +frame_id: '' and grasped_object_mask header seq: 70060 +stamp: + secs: 1735525736 + nsecs: 907131195 +frame_id: "camera_color_optical_frame" +[INFO] [1735525737.322236]: received mask_obs with header seq: 629 +stamp: + secs: 1735525737 + nsecs: 323616981 +frame_id: '' and grasped_object_mask header seq: 70066 +stamp: + secs: 1735525737 + nsecs: 106727123 +frame_id: "camera_color_optical_frame" +[INFO] [1735525737.518922]: received mask_obs with header seq: 630 +stamp: + secs: 1735525737 + nsecs: 519895792 +frame_id: '' and grasped_object_mask header seq: 70072 +stamp: + secs: 1735525737 + nsecs: 306311369 +frame_id: "camera_color_optical_frame" +[INFO] [1735525737.719903]: received mask_obs with header seq: 631 +stamp: + secs: 1735525737 + nsecs: 719918012 +frame_id: '' and grasped_object_mask header seq: 70078 +stamp: + secs: 1735525737 + nsecs: 505883932 +frame_id: "camera_color_optical_frame" +[INFO] [1735525737.919573]: received mask_obs with header seq: 632 +stamp: + secs: 1735525737 + nsecs: 919035911 +frame_id: '' and grasped_object_mask header seq: 70084 +stamp: + secs: 1735525737 + nsecs: 705474377 +frame_id: "camera_color_optical_frame" +[INFO] [1735525738.119196]: received mask_obs with header seq: 633 +stamp: + secs: 1735525738 + nsecs: 118971824 +frame_id: '' and grasped_object_mask header seq: 70090 +stamp: + secs: 1735525737 + nsecs: 905081987 +frame_id: "camera_color_optical_frame" +[INFO] [1735525738.319402]: received mask_obs with header seq: 634 +stamp: + secs: 1735525738 + nsecs: 318860769 +frame_id: '' and grasped_object_mask header seq: 70096 +stamp: + secs: 1735525738 + nsecs: 104679108 +frame_id: "camera_color_optical_frame" +[INFO] [1735525738.518730]: received mask_obs with header seq: 635 +stamp: + secs: 1735525738 + nsecs: 518709659 +frame_id: '' and grasped_object_mask header seq: 70102 +stamp: + secs: 1735525738 + nsecs: 304263592 +frame_id: "camera_color_optical_frame" +[INFO] [1735525738.717084]: received mask_obs with header seq: 636 +stamp: + secs: 1735525738 + nsecs: 718329191 +frame_id: '' and grasped_object_mask header seq: 70108 +stamp: + secs: 1735525738 + nsecs: 503836393 +frame_id: "camera_color_optical_frame" +[INFO] [1735525738.920378]: received mask_obs with header seq: 637 +stamp: + secs: 1735525738 + nsecs: 921811342 +frame_id: '' and grasped_object_mask header seq: 70114 +stamp: + secs: 1735525738 + nsecs: 703426361 +frame_id: "camera_color_optical_frame" +[INFO] [1735525739.118098]: received mask_obs with header seq: 638 +stamp: + secs: 1735525739 + nsecs: 119444847 +frame_id: '' and grasped_object_mask header seq: 70120 +stamp: + secs: 1735525738 + nsecs: 903033018 +frame_id: "camera_color_optical_frame" +[INFO] [1735525739.317777]: received mask_obs with header seq: 639 +stamp: + secs: 1735525739 + nsecs: 318187713 +frame_id: '' and grasped_object_mask header seq: 70126 +stamp: + secs: 1735525739 + nsecs: 102627993 +frame_id: "camera_color_optical_frame" +[INFO] [1735525739.517629]: received mask_obs with header seq: 640 +stamp: + secs: 1735525739 + nsecs: 518529176 +frame_id: '' and grasped_object_mask header seq: 70132 +stamp: + secs: 1735525739 + nsecs: 302212238 +frame_id: "camera_color_optical_frame" +[INFO] [1735525739.717115]: received mask_obs with header seq: 641 +stamp: + secs: 1735525739 + nsecs: 717705726 +frame_id: '' and grasped_object_mask header seq: 70138 +stamp: + secs: 1735525739 + nsecs: 501783609 +frame_id: "camera_color_optical_frame" +[INFO] [1735525739.917989]: received mask_obs with header seq: 642 +stamp: + secs: 1735525739 + nsecs: 917566537 +frame_id: '' and grasped_object_mask header seq: 70144 +stamp: + secs: 1735525739 + nsecs: 701373100 +frame_id: "camera_color_optical_frame" +[INFO] [1735525740.121168]: received mask_obs with header seq: 643 +stamp: + secs: 1735525740 + nsecs: 122226476 +frame_id: '' and grasped_object_mask header seq: 70150 +stamp: + secs: 1735525739 + nsecs: 900980234 +frame_id: "camera_color_optical_frame" +[INFO] [1735525740.317734]: received mask_obs with header seq: 644 +stamp: + secs: 1735525740 + nsecs: 318905830 +frame_id: '' and grasped_object_mask header seq: 70156 +stamp: + secs: 1735525740 + nsecs: 100575447 +frame_id: "camera_color_optical_frame" +[INFO] [1735525740.529713]: received mask_obs with header seq: 645 +stamp: + secs: 1735525740 + nsecs: 531114578 +frame_id: '' and grasped_object_mask header seq: 70162 +stamp: + secs: 1735525740 + nsecs: 300159216 +frame_id: "camera_color_optical_frame" +[INFO] [1735525740.724412]: received mask_obs with header seq: 646 +stamp: + secs: 1735525740 + nsecs: 725538492 +frame_id: '' and grasped_object_mask header seq: 70168 +stamp: + secs: 1735525740 + nsecs: 499732018 +frame_id: "camera_color_optical_frame" +[INFO] [1735525740.917936]: received mask_obs with header seq: 647 +stamp: + secs: 1735525740 + nsecs: 917798757 +frame_id: '' and grasped_object_mask header seq: 70174 +stamp: + secs: 1735525740 + nsecs: 699320555 +frame_id: "camera_color_optical_frame" +[INFO] [1735525741.119189]: received mask_obs with header seq: 648 +stamp: + secs: 1735525741 + nsecs: 120068550 +frame_id: '' and grasped_object_mask header seq: 70180 +stamp: + secs: 1735525740 + nsecs: 898926020 +frame_id: "camera_color_optical_frame" +[INFO] [1735525741.315004]: received mask_obs with header seq: 649 +stamp: + secs: 1735525741 + nsecs: 316268444 +frame_id: '' and grasped_object_mask header seq: 70186 +stamp: + secs: 1735525741 + nsecs: 98519802 +frame_id: "camera_color_optical_frame" +[INFO] [1735525741.517335]: received mask_obs with header seq: 650 +stamp: + secs: 1735525741 + nsecs: 516336441 +frame_id: '' and grasped_object_mask header seq: 70192 +stamp: + secs: 1735525741 + nsecs: 298101664 +frame_id: "camera_color_optical_frame" +[INFO] [1735525741.716767]: received mask_obs with header seq: 651 +stamp: + secs: 1735525741 + nsecs: 716760158 +frame_id: '' and grasped_object_mask header seq: 70198 +stamp: + secs: 1735525741 + nsecs: 497671843 +frame_id: "camera_color_optical_frame" +[INFO] [1735525741.916453]: received mask_obs with header seq: 652 +stamp: + secs: 1735525741 + nsecs: 916044712 +frame_id: '' and grasped_object_mask header seq: 70204 +stamp: + secs: 1735525741 + nsecs: 697256804 +frame_id: "camera_color_optical_frame" +[INFO] [1735525742.118641]: received mask_obs with header seq: 653 +stamp: + secs: 1735525742 + nsecs: 120075702 +frame_id: '' and grasped_object_mask header seq: 70210 +stamp: + secs: 1735525741 + nsecs: 896857500 +frame_id: "camera_color_optical_frame" +[INFO] [1735525742.316593]: received mask_obs with header seq: 654 +stamp: + secs: 1735525742 + nsecs: 316240310 +frame_id: '' and grasped_object_mask header seq: 70216 +stamp: + secs: 1735525742 + nsecs: 96444845 +frame_id: "camera_color_optical_frame" +[INFO] [1735525742.523712]: received mask_obs with header seq: 655 +stamp: + secs: 1735525742 + nsecs: 524720430 +frame_id: '' and grasped_object_mask header seq: 70222 +stamp: + secs: 1735525742 + nsecs: 296020269 +frame_id: "camera_color_optical_frame" +[INFO] [1735525742.716856]: received mask_obs with header seq: 656 +stamp: + secs: 1735525742 + nsecs: 716390609 +frame_id: '' and grasped_object_mask header seq: 70228 +stamp: + secs: 1735525742 + nsecs: 495583296 +frame_id: "camera_color_optical_frame" +[INFO] [1735525742.916762]: received mask_obs with header seq: 657 +stamp: + secs: 1735525742 + nsecs: 917933940 +frame_id: '' and grasped_object_mask header seq: 70234 +stamp: + secs: 1735525742 + nsecs: 695165157 +frame_id: "camera_color_optical_frame" +[INFO] [1735525743.115499]: received mask_obs with header seq: 658 +stamp: + secs: 1735525743 + nsecs: 116929769 +frame_id: '' and grasped_object_mask header seq: 70240 +stamp: + secs: 1735525742 + nsecs: 894766092 +frame_id: "camera_color_optical_frame" +[INFO] [1735525743.316556]: received mask_obs with header seq: 659 +stamp: + secs: 1735525743 + nsecs: 317904710 +frame_id: '' and grasped_object_mask header seq: 70246 +stamp: + secs: 1735525743 + nsecs: 94356060 +frame_id: "camera_color_optical_frame" +[INFO] [1735525743.514462]: received mask_obs with header seq: 660 +stamp: + secs: 1735525743 + nsecs: 515868425 +frame_id: '' and grasped_object_mask header seq: 70252 +stamp: + secs: 1735525743 + nsecs: 293932676 +frame_id: "camera_color_optical_frame" +[INFO] [1735525743.716577]: received mask_obs with header seq: 661 +stamp: + secs: 1735525743 + nsecs: 718007564 +frame_id: '' and grasped_object_mask header seq: 70258 +stamp: + secs: 1735525743 + nsecs: 493497372 +frame_id: "camera_color_optical_frame" +[INFO] [1735525743.915011]: received mask_obs with header seq: 662 +stamp: + secs: 1735525743 + nsecs: 915385484 +frame_id: '' and grasped_object_mask header seq: 70264 +stamp: + secs: 1735525743 + nsecs: 693082333 +frame_id: "camera_color_optical_frame" +[INFO] [1735525744.116617]: received mask_obs with header seq: 663 +stamp: + secs: 1735525744 + nsecs: 118072032 +frame_id: '' and grasped_object_mask header seq: 70270 +stamp: + secs: 1735525743 + nsecs: 892687559 +frame_id: "camera_color_optical_frame" +[INFO] [1735525744.314299]: received mask_obs with header seq: 664 +stamp: + secs: 1735525744 + nsecs: 315538406 +frame_id: '' and grasped_object_mask header seq: 70276 +stamp: + secs: 1735525744 + nsecs: 92280865 +frame_id: "camera_color_optical_frame" +[INFO] [1735525744.344378]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1735525745.171134]: Stored episode 5. +[INFO] [1735525745.174204]: Saving video... +[INFO] [1735525745.510755]: Video saved! +[INFO] [1735525745.558953]: resetting environment +[INFO] [1735525745.602888]: cleared current plan +[INFO] [1735525745.606005]: moving to home +[WARN] [1735525745.507355]: Plan exhausted +[WARN] [1735525745.538919]: Plan exhausted +[WARN] [1735525745.580665]: Plan exhausted +[INFO] [1735525749.225866]: reached home +[INFO] [1735525749.235926]: reset action history +[INFO] [1735525752.084751]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! \ No newline at end of file diff --git a/212506/wandb/run-20241229_212512-oco0rjll/files/requirements.txt b/212506/wandb/run-20241229_212512-oco0rjll/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..192179a195956e252bda33f2342b4970a9de79d9 --- /dev/null +++ b/212506/wandb/run-20241229_212512-oco0rjll/files/requirements.txt @@ -0,0 +1,339 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.2 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-metadata.json b/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..da433b0add8343824b5e4c2d5c668fc1121bf814 --- /dev/null +++ b/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2024-12-30T02:25:13.109189", + "startedAt": "2024-12-30T02:25:12.622611", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "23581857e509febdae7f0ac45cc9cf4f9081f85c" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3762.1391249999997, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4520.366, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4546.057, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 477.0821304321289 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json b/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e270c303c58afbf5a9bdcf407bc26fea3075222a --- /dev/null +++ b/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "7f4ade8f7a09940de9a392152321c741514d0b2b83b953d2bc85b2587d34b370", "size": 1160403, "path": "media/videos/eval/0_eval_0_7f4ade8f7a09940de9a3.mp4"}, "global_step": 664, "_timestamp": 1735525754.6580799, "_runtime": 242.02804493904114, "_step": 9, "eval/1_eval": {"_type": "video-file", "sha256": "df2b523b5c69376abbb661917fe8d60d3ad79f771878de2226ab5c89077a3630", "size": 1160649, "path": "media/videos/eval/1_eval_1_df2b523b5c69376abbb6.mp4"}, "eval/num_success": 5.0, "episode": 5.0, "eval/success_rate": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "56287c330ba45a21ceae2f5ada5b8060ed510846ec2d5a3b90730f83caff3226", "size": 1157924, "path": "media/videos/eval/2_eval_2_56287c330ba45a21ceae.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "d9096d3d3b1128c5f3af34fcf97e720fde5df69b1403591cedf85594195eab69", "size": 1175711, "path": "media/videos/eval/3_eval_3_d9096d3d3b1128c5f3af.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "277cb7c2f38e9f60cb6a1957cbae5c162329b3ce1d0991b1503aabd5e0830c0b", "size": 1163227, "path": "media/videos/eval/4_eval_4_277cb7c2f38e9f60cb6a.mp4"}, "_wandb": {"runtime": 241}} \ No newline at end of file diff --git a/212506/wandb/run-20241229_212512-oco0rjll/logs/debug-internal.log b/212506/wandb/run-20241229_212512-oco0rjll/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..471a85fc44a690cc015228dd46677c7cd6235565 --- /dev/null +++ b/212506/wandb/run-20241229_212512-oco0rjll/logs/debug-internal.log @@ -0,0 +1,736 @@ +2024-12-29 21:25:12,629 INFO StreamThr :1360479 [internal.py:wandb_internal():85] W&B internal server running at pid: 1360479, started at: 2024-12-29 21:25:12.629472 +2024-12-29 21:25:12,631 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status +2024-12-29 21:25:12,641 INFO WriterThread:1360479 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/run-oco0rjll.wandb +2024-12-29 21:25:12,641 DEBUG SenderThread:1360479 [sender.py:send():379] send: header +2024-12-29 21:25:12,651 DEBUG SenderThread:1360479 [sender.py:send():379] send: run +2024-12-29 21:25:12,989 INFO SenderThread:1360479 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files +2024-12-29 21:25:12,989 INFO SenderThread:1360479 [sender.py:_start_run_threads():1188] run started: oco0rjll with start time 1735525512.630035 +2024-12-29 21:25:13,005 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: check_version +2024-12-29 21:25:13,006 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: check_version +2024-12-29 21:25:13,088 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: run_start +2024-12-29 21:25:13,093 DEBUG HandlerThread:1360479 [system_info.py:__init__():26] System info init +2024-12-29 21:25:13,093 DEBUG HandlerThread:1360479 [system_info.py:__init__():41] System info init done +2024-12-29 21:25:13,093 INFO HandlerThread:1360479 [system_monitor.py:start():194] Starting system monitor +2024-12-29 21:25:13,094 INFO SystemMonitor:1360479 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-29 21:25:13,094 INFO HandlerThread:1360479 [system_monitor.py:probe():214] Collecting system info +2024-12-29 21:25:13,094 INFO SystemMonitor:1360479 [interfaces.py:start():188] Started cpu monitoring +2024-12-29 21:25:13,095 INFO SystemMonitor:1360479 [interfaces.py:start():188] Started disk monitoring +2024-12-29 21:25:13,095 INFO SystemMonitor:1360479 [interfaces.py:start():188] Started gpu monitoring +2024-12-29 21:25:13,096 INFO SystemMonitor:1360479 [interfaces.py:start():188] Started memory monitoring +2024-12-29 21:25:13,096 INFO SystemMonitor:1360479 [interfaces.py:start():188] Started network monitoring +2024-12-29 21:25:13,109 DEBUG HandlerThread:1360479 [system_info.py:probe():152] Probing system +2024-12-29 21:25:13,110 DEBUG HandlerThread:1360479 [system_info.py:_probe_git():137] Probing git +2024-12-29 21:25:13,115 DEBUG HandlerThread:1360479 [system_info.py:_probe_git():145] Probing git done +2024-12-29 21:25:13,115 DEBUG HandlerThread:1360479 [system_info.py:probe():200] Probing system done +2024-12-29 21:25:13,115 DEBUG HandlerThread:1360479 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-30T02:25:13.109189', 'startedAt': '2024-12-30T02:25:12.622611', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '23581857e509febdae7f0ac45cc9cf4f9081f85c'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3762.1391249999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4520.366, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4546.057, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 477.0821304321289}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-29 21:25:13,115 INFO HandlerThread:1360479 [system_monitor.py:probe():224] Finished collecting system info +2024-12-29 21:25:13,115 INFO HandlerThread:1360479 [system_monitor.py:probe():227] Publishing system info +2024-12-29 21:25:13,115 DEBUG HandlerThread:1360479 [system_info.py:_save_code():44] Saving code +2024-12-29 21:25:13,116 DEBUG HandlerThread:1360479 [system_info.py:_save_code():67] Saving code done +2024-12-29 21:25:13,116 DEBUG HandlerThread:1360479 [system_info.py:_save_patches():84] Saving git patches +2024-12-29 21:25:13,179 DEBUG HandlerThread:1360479 [system_info.py:_save_patches():126] Saving git patches done +2024-12-29 21:25:13,179 INFO HandlerThread:1360479 [system_monitor.py:probe():229] Finished publishing system info +2024-12-29 21:25:13,182 DEBUG SenderThread:1360479 [sender.py:send():379] send: files +2024-12-29 21:25:13,182 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-29 21:25:13,183 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-29 21:25:13,183 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-29 21:25:13,418 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: python_packages +2024-12-29 21:25:13,418 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: python_packages +2024-12-29 21:25:13,419 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:25:13,419 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:25:13,420 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:13,422 INFO HandlerThread:1360479 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/tb" + save: true +} +control { + relay_id: "371900e530ae4f619f94a7297cc74e9c" +} +_info { + stream_id: "oco0rjll" +} + +2024-12-29 21:25:13,422 DEBUG HandlerThread:1360479 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-29 21:25:13,452 INFO wandb-upload_2:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/97vjqa7c-diff.patch +2024-12-29 21:25:13,476 INFO wandb-upload_1:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/1k2sknsw-code/FISH/eval_robot.py +2024-12-29 21:25:13,479 INFO wandb-upload_0:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/qurbt2cg-wandb-metadata.json +2024-12-29 21:25:13,521 DEBUG SenderThread:1360479 [sender.py:send():379] send: telemetry +2024-12-29 21:25:13,521 DEBUG SenderThread:1360479 [sender.py:send():379] send: tbrecord +2024-12-29 21:25:13,522 DEBUG SenderThread:1360479 [sender.py:send():379] send: files +2024-12-29 21:25:13,522 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 with policy live +2024-12-29 21:25:13,710 INFO wandb-upload_2:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/f1ksd97h-events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 +2024-12-29 21:25:13,991 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/requirements.txt +2024-12-29 21:25:13,991 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/code/FISH/eval_robot.py +2024-12-29 21:25:13,991 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:13,991 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/diff.patch +2024-12-29 21:25:13,991 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-metadata.json +2024-12-29 21:25:13,992 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 +2024-12-29 21:25:13,992 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/code +2024-12-29 21:25:13,992 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/code/FISH +2024-12-29 21:25:14,419 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:15,420 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:15,990 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:16,420 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:17,420 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:17,663 DEBUG SenderThread:1360479 [sender.py:send():379] send: config +2024-12-29 21:25:17,663 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:25:18,420 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:19,421 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:19,991 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:20,421 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:21,425 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:22,433 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:23,419 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:25:23,421 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:23,992 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:24,421 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:25,423 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:26,423 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:27,423 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:27,993 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:28,418 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:25:28,419 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:25:28,461 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:28,526 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:25:29,423 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:30,423 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:31,423 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:32,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:33,423 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:34,419 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:25:34,423 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:35,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:36,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:37,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:37,996 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:38,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:39,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:39,997 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:40,419 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:25:40,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:41,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:41,997 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:42,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:43,419 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:25:43,419 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:25:43,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:43,998 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:44,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:45,424 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:45,603 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:25:45,998 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:45,999 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/config.yaml +2024-12-29 21:25:46,425 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:47,425 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:47,999 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:48,425 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:49,425 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:49,999 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:50,425 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:50,804 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:25:51,425 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:52,000 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:52,425 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:53,425 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:54,001 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:54,426 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:55,426 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:55,997 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:25:56,001 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:56,426 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:57,426 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:58,002 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:25:58,419 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:25:58,419 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:25:58,426 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:25:59,427 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:00,002 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:00,427 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:01,003 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:01,428 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:02,004 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:02,428 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:03,428 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:04,004 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:04,428 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:05,428 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:06,005 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:06,194 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:06,429 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:07,429 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:08,005 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:08,429 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:08,601 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 21:26:08,602 DEBUG SenderThread:1360479 [sender.py:send():379] send: files +2024-12-29 21:26:08,602 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_7f4ade8f7a09940de9a3.mp4 with policy now +2024-12-29 21:26:08,602 DEBUG SenderThread:1360479 [sender.py:send():379] send: history +2024-12-29 21:26:08,602 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:26:08,603 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:26:08,857 INFO wandb-upload_0:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/r82fuotw-media/videos/eval/0_eval_0_7f4ade8f7a09940de9a3.mp4 +2024-12-29 21:26:09,006 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json +2024-12-29 21:26:09,006 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/0_eval_0_7f4ade8f7a09940de9a3.mp4 +2024-12-29 21:26:09,006 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos +2024-12-29 21:26:09,006 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval +2024-12-29 21:26:09,006 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media +2024-12-29 21:26:09,266 INFO wandb-upload_2:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/4nfn6llk-media/videos/eval/0_eval_0_7f4ade8f7a09940de9a3.mp4 +2024-12-29 21:26:09,429 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:10,006 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:10,430 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:11,430 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:11,603 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:12,430 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:13,097 DEBUG SystemMonitor:1360479 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-29 21:26:13,097 DEBUG SenderThread:1360479 [sender.py:send():379] send: stats +2024-12-29 21:26:13,420 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:26:13,421 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:26:13,461 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:14,007 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:14,430 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:15,430 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:16,008 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:16,431 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:16,934 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:17,431 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:18,009 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 +2024-12-29 21:26:18,259 INFO wandb-upload_0:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/znjalm43-events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 +2024-12-29 21:26:18,431 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:19,431 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:20,010 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:20,431 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:21,431 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:21,965 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:22,432 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:23,431 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:24,011 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:24,432 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:25,432 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:26,012 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:26,432 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:27,006 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:27,432 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:28,012 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:28,419 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:26:28,419 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:26:28,432 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:29,432 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:30,013 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:30,432 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:31,432 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:32,014 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:32,202 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:32,432 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:33,433 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:34,015 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:34,433 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:35,433 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:36,015 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:36,434 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:37,204 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:37,434 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:38,016 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:38,434 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:39,434 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:40,017 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:40,434 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:41,435 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:42,018 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:42,396 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:42,435 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:43,098 DEBUG SenderThread:1360479 [sender.py:send():379] send: stats +2024-12-29 21:26:43,419 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:26:43,420 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:26:43,435 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:44,018 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:44,435 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:45,435 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:46,019 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:46,435 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:47,397 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:47,435 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:48,019 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:48,435 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:49,437 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:49,687 DEBUG SenderThread:1360479 [sender.py:send():379] send: files +2024-12-29 21:26:49,687 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_df2b523b5c69376abbb6.mp4 with policy now +2024-12-29 21:26:49,688 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 21:26:49,688 DEBUG SenderThread:1360479 [sender.py:send():379] send: history +2024-12-29 21:26:49,689 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:26:49,689 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:26:49,933 INFO wandb-upload_2:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/epacwhln-media/videos/eval/1_eval_1_df2b523b5c69376abbb6.mp4 +2024-12-29 21:26:50,020 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json +2024-12-29 21:26:50,020 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:50,021 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/1_eval_1_df2b523b5c69376abbb6.mp4 +2024-12-29 21:26:50,021 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval +2024-12-29 21:26:50,292 INFO wandb-upload_0:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/uuduyhwz-media/videos/eval/1_eval_1_df2b523b5c69376abbb6.mp4 +2024-12-29 21:26:50,438 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:51,451 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:52,021 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:52,447 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:52,709 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:53,444 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:54,446 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:55,445 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:56,022 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:56,440 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:57,440 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:58,023 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:26:58,133 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:26:58,437 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:26:58,438 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:26:58,440 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:26:58,807 DEBUG SenderThread:1360479 [sender.py:send():379] send: history +2024-12-29 21:26:58,807 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:26:58,812 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:26:59,023 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json +2024-12-29 21:26:59,023 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 +2024-12-29 21:26:59,449 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:00,023 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:00,441 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:01,441 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:02,024 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:02,441 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:03,225 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:27:03,441 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:04,025 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:04,441 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:05,441 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:06,026 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:06,441 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:07,442 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:08,027 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:08,238 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:27:08,481 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:09,442 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:10,027 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:10,442 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:11,442 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:12,028 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:12,442 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:13,098 DEBUG SenderThread:1360479 [sender.py:send():379] send: stats +2024-12-29 21:27:13,420 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:27:13,420 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:27:13,442 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:13,532 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:27:14,029 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:14,442 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:15,443 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:16,030 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:16,443 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:17,443 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:18,030 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:18,485 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:18,642 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:27:19,443 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:20,031 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:20,443 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:21,443 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:22,032 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:22,443 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:23,481 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:23,837 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:27:24,032 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:24,444 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:25,444 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:26,033 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:26,444 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:27,444 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:28,034 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:28,420 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:27:28,421 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:27:28,444 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:29,035 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:27:29,456 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:30,035 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:30,357 DEBUG SenderThread:1360479 [sender.py:send():379] send: files +2024-12-29 21:27:30,358 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_56287c330ba45a21ceae.mp4 with policy now +2024-12-29 21:27:30,363 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 21:27:30,364 DEBUG SenderThread:1360479 [sender.py:send():379] send: history +2024-12-29 21:27:30,364 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:27:30,365 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:27:30,456 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:30,610 INFO wandb-upload_2:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/of9cesez-media/videos/eval/2_eval_2_56287c330ba45a21ceae.mp4 +2024-12-29 21:27:31,035 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json +2024-12-29 21:27:31,035 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/2_eval_2_56287c330ba45a21ceae.mp4 +2024-12-29 21:27:31,035 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval +2024-12-29 21:27:31,310 INFO wandb-upload_0:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/p7ulzqh4-media/videos/eval/2_eval_2_56287c330ba45a21ceae.mp4 +2024-12-29 21:27:31,457 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:32,035 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:32,452 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:33,452 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:34,073 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:27:34,454 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:35,455 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:36,037 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:36,466 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:37,457 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:38,037 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:38,453 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:39,445 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:27:39,453 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:40,456 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:41,457 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:42,465 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:43,099 DEBUG SenderThread:1360479 [sender.py:send():379] send: stats +2024-12-29 21:27:43,420 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:27:43,421 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:27:43,466 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:44,468 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:44,552 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:27:45,040 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 +2024-12-29 21:27:45,286 INFO wandb-upload_2:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/w7589ncm-events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 +2024-12-29 21:27:45,454 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:45,649 DEBUG SenderThread:1360479 [sender.py:send():379] send: history +2024-12-29 21:27:45,649 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:27:45,649 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:27:46,040 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json +2024-12-29 21:27:46,040 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:46,455 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:47,460 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:48,041 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:48,467 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:49,458 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:49,834 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:27:50,041 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:50,459 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:51,459 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:52,042 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:52,459 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:53,497 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:54,043 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:54,459 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:54,870 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:27:55,459 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:56,043 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:56,459 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:57,460 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:58,044 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:27:58,420 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:27:58,420 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:27:58,460 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:59,460 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:27:59,941 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:00,045 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:00,460 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:01,460 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:02,046 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:02,460 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:03,501 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:04,046 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:04,462 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:05,062 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:05,462 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:06,047 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:06,462 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:07,462 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:08,048 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:08,509 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:09,463 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:10,049 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:10,256 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:10,462 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:11,463 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:12,049 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:12,463 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:13,099 DEBUG SenderThread:1360479 [sender.py:send():379] send: stats +2024-12-29 21:28:13,420 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:28:13,420 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:28:13,463 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:14,050 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:14,463 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:15,263 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:15,463 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:16,051 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:16,472 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:17,233 DEBUG SenderThread:1360479 [sender.py:send():379] send: files +2024-12-29 21:28:17,234 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_d9096d3d3b1128c5f3af.mp4 with policy now +2024-12-29 21:28:17,238 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 21:28:17,239 DEBUG SenderThread:1360479 [sender.py:send():379] send: history +2024-12-29 21:28:17,239 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:28:17,239 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:28:17,466 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:17,502 INFO wandb-upload_0:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/09zme4ma-media/videos/eval/3_eval_3_d9096d3d3b1128c5f3af.mp4 +2024-12-29 21:28:18,051 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json +2024-12-29 21:28:18,052 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:18,052 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/3_eval_3_d9096d3d3b1128c5f3af.mp4 +2024-12-29 21:28:18,052 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval +2024-12-29 21:28:18,332 INFO wandb-upload_2:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/8rm19avh-media/videos/eval/3_eval_3_d9096d3d3b1128c5f3af.mp4 +2024-12-29 21:28:18,475 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:19,477 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:20,457 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:20,481 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:21,477 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:22,053 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:22,489 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:23,483 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:24,483 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:25,481 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:25,785 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:26,054 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:26,485 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:27,483 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:28,430 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:28:28,430 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:28:28,489 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:29,482 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:30,481 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:31,483 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:31,580 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:32,490 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:33,487 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:33,701 DEBUG SenderThread:1360479 [sender.py:send():379] send: history +2024-12-29 21:28:33,701 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:28:33,701 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:28:34,057 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json +2024-12-29 21:28:34,057 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:34,057 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 +2024-12-29 21:28:34,492 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:35,493 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:36,057 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:36,502 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:37,009 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:37,508 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:38,058 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:38,541 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:39,507 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:40,059 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:40,507 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:41,507 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:42,060 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:42,119 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:42,507 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:43,100 DEBUG SenderThread:1360479 [sender.py:send():379] send: stats +2024-12-29 21:28:43,427 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:28:43,427 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:28:43,507 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:44,060 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:44,507 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:45,508 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:46,061 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:46,508 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:47,125 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:47,508 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:48,062 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:48,508 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:49,508 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:50,063 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:50,508 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:51,508 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:52,064 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:52,130 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:52,509 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:53,508 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:54,064 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:54,509 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:55,509 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:56,065 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:56,509 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:57,322 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:28:57,509 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:58,066 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:28:58,427 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:28:58,428 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:28:58,509 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:28:59,509 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:00,067 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:29:00,509 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:01,509 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:02,067 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:29:02,509 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:02,524 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:29:03,510 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:04,068 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:29:04,510 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:05,517 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:05,589 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 21:29:05,589 DEBUG SenderThread:1360479 [sender.py:send():379] send: files +2024-12-29 21:29:05,589 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_277cb7c2f38e9f60cb6a.mp4 with policy now +2024-12-29 21:29:05,590 DEBUG SenderThread:1360479 [sender.py:send():379] send: history +2024-12-29 21:29:05,590 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:29:05,590 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:29:05,834 INFO wandb-upload_0:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/6o3pw2m5-media/videos/eval/4_eval_4_277cb7c2f38e9f60cb6a.mp4 +2024-12-29 21:29:06,069 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json +2024-12-29 21:29:06,069 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:29:06,069 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/4_eval_4_277cb7c2f38e9f60cb6a.mp4 +2024-12-29 21:29:06,069 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval +2024-12-29 21:29:06,324 INFO wandb-upload_2:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/njaqymsx-media/videos/eval/4_eval_4_277cb7c2f38e9f60cb6a.mp4 +2024-12-29 21:29:06,517 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:07,528 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:07,611 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:29:08,069 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:29:08,520 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:09,520 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:10,070 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:29:10,538 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:11,525 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:12,530 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:13,100 DEBUG SenderThread:1360479 [sender.py:send():379] send: stats +2024-12-29 21:29:13,101 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:29:13,443 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 21:29:13,444 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: stop_status +2024-12-29 21:29:13,549 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:14,072 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:29:14,539 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:14,706 DEBUG SenderThread:1360479 [sender.py:send():379] send: exit +2024-12-29 21:29:14,706 INFO SenderThread:1360479 [sender.py:send_exit():586] handling exit code: 0 +2024-12-29 21:29:14,706 INFO SenderThread:1360479 [sender.py:send_exit():588] handling runtime: 241 +2024-12-29 21:29:14,721 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:29:14,721 INFO SenderThread:1360479 [sender.py:send_exit():594] send defer +2024-12-29 21:29:14,721 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:14,721 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-29 21:29:14,722 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:14,722 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-29 21:29:14,722 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 1 +2024-12-29 21:29:14,722 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:14,722 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-29 21:29:14,722 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:14,722 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-29 21:29:14,722 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 2 +2024-12-29 21:29:14,722 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:14,722 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-29 21:29:14,722 INFO HandlerThread:1360479 [system_monitor.py:finish():203] Stopping system monitor +2024-12-29 21:29:14,722 DEBUG SystemMonitor:1360479 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-29 21:29:14,722 INFO HandlerThread:1360479 [interfaces.py:finish():200] Joined cpu monitor +2024-12-29 21:29:14,722 DEBUG SystemMonitor:1360479 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-29 21:29:14,723 INFO HandlerThread:1360479 [interfaces.py:finish():200] Joined disk monitor +2024-12-29 21:29:14,773 INFO HandlerThread:1360479 [interfaces.py:finish():200] Joined gpu monitor +2024-12-29 21:29:14,774 INFO HandlerThread:1360479 [interfaces.py:finish():200] Joined memory monitor +2024-12-29 21:29:14,774 INFO HandlerThread:1360479 [interfaces.py:finish():200] Joined network monitor +2024-12-29 21:29:14,774 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:14,774 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-29 21:29:14,774 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 3 +2024-12-29 21:29:14,775 DEBUG SenderThread:1360479 [sender.py:send():379] send: stats +2024-12-29 21:29:14,775 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:14,775 DEBUG SenderThread:1360479 [sender.py:send():379] send: history +2024-12-29 21:29:14,775 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-29 21:29:14,775 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:29:14,796 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:29:14,797 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:14,797 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-29 21:29:14,797 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 4 +2024-12-29 21:29:14,797 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:14,797 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-29 21:29:15,072 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json +2024-12-29 21:29:15,072 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 +2024-12-29 21:29:15,347 INFO wandb-upload_0:1360479 [upload_job.py:push():130] Uploaded file /tmp/tmpkjk2br4wwandb/mr7i954z-events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 +2024-12-29 21:29:16,072 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:29:21,759 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:29:21,759 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:29:21,759 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:21,759 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: keepalive +2024-12-29 21:29:21,759 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-29 21:29:21,759 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 5 +2024-12-29 21:29:21,759 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:29:21,760 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:21,760 DEBUG SenderThread:1360479 [sender.py:send():379] send: history +2024-12-29 21:29:21,760 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-29 21:29:21,760 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: summary_record +2024-12-29 21:29:21,760 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:29:21,760 DEBUG SenderThread:1360479 [sender.py:send():379] send: summary +2024-12-29 21:29:21,761 INFO SenderThread:1360479 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 21:29:21,761 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:21,761 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-29 21:29:21,761 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 6 +2024-12-29 21:29:21,761 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:21,761 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-29 21:29:21,761 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:21,761 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-29 21:29:21,761 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 7 +2024-12-29 21:29:21,761 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 21:29:21,761 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:21,762 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-29 21:29:21,762 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:21,762 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-29 21:29:22,074 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json +2024-12-29 21:29:22,707 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:29:23,721 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 8 +2024-12-29 21:29:23,721 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:29:23,721 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:23,721 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-29 21:29:23,722 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:23,722 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-29 21:29:23,722 INFO SenderThread:1360479 [job_builder.py:build():440] Attempting to build job artifact +2024-12-29 21:29:23,724 INFO SenderThread:1360479 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-29 21:29:23,738 INFO SenderThread:1360479 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-29 21:29:23,752 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 9 +2024-12-29 21:29:23,753 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:23,753 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-29 21:29:23,753 DEBUG SenderThread:1360479 [sender.py:send():379] send: artifact +2024-12-29 21:29:24,075 INFO Thread-12 :1360479 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:29:24,192 INFO wandb-upload_1:1360479 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpu0f2y331 +2024-12-29 21:29:24,192 INFO wandb-upload_0:1360479 [upload_job.py:push():85] Skipped uploading /tmp/tmp6dzmspxx/wandb-job.json +2024-12-29 21:29:24,336 INFO wandb-upload_2:1360479 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpf1m3m3qp +2024-12-29 21:29:24,708 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:29:24,896 INFO SenderThread:1360479 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxMjA3MTYzNw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxMjA2MzIwOA==', 'versionIndex': 387}}} +2024-12-29 21:29:24,896 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:24,896 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-29 21:29:24,896 INFO SenderThread:1360479 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-29 21:29:25,076 INFO SenderThread:1360479 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files +2024-12-29 21:29:25,076 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/config.yaml config.yaml +2024-12-29 21:29:25,076 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json wandb-summary.json +2024-12-29 21:29:25,077 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/requirements.txt requirements.txt +2024-12-29 21:29:25,078 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-metadata.json wandb-metadata.json +2024-12-29 21:29:25,078 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 events.out.tfevents.1735525513.leonmkim-ROG-Strix-G15CS-G15CS.1360290.0 +2024-12-29 21:29:25,079 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log output.log +2024-12-29 21:29:25,079 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/diff.patch diff.patch +2024-12-29 21:29:25,083 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-29 21:29:25,084 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/1_eval_1_df2b523b5c69376abbb6.mp4 media/videos/eval/1_eval_1_df2b523b5c69376abbb6.mp4 +2024-12-29 21:29:25,084 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/2_eval_2_56287c330ba45a21ceae.mp4 media/videos/eval/2_eval_2_56287c330ba45a21ceae.mp4 +2024-12-29 21:29:25,084 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/4_eval_4_277cb7c2f38e9f60cb6a.mp4 media/videos/eval/4_eval_4_277cb7c2f38e9f60cb6a.mp4 +2024-12-29 21:29:25,084 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/3_eval_3_d9096d3d3b1128c5f3af.mp4 media/videos/eval/3_eval_3_d9096d3d3b1128c5f3af.mp4 +2024-12-29 21:29:25,084 INFO SenderThread:1360479 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/media/videos/eval/0_eval_0_7f4ade8f7a09940de9a3.mp4 media/videos/eval/0_eval_0_7f4ade8f7a09940de9a3.mp4 +2024-12-29 21:29:25,084 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 10 +2024-12-29 21:29:25,084 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:29:25,084 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:25,084 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-29 21:29:25,085 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:25,085 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-29 21:29:25,085 INFO SenderThread:1360479 [file_pusher.py:finish():169] shutting down file pusher +2024-12-29 21:29:25,270 INFO wandb-upload_2:1360479 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/wandb-summary.json +2024-12-29 21:29:25,292 INFO wandb-upload_3:1360479 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/output.log +2024-12-29 21:29:25,297 INFO wandb-upload_1:1360479 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/config.yaml +2024-12-29 21:29:25,317 INFO wandb-upload_0:1360479 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/files/requirements.txt +2024-12-29 21:29:25,517 INFO Thread-11 (_thread_body):1360479 [sender.py:transition_state():614] send defer: 11 +2024-12-29 21:29:25,517 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:25,517 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-29 21:29:25,517 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:25,517 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-29 21:29:25,517 INFO SenderThread:1360479 [file_pusher.py:join():175] waiting for file pusher +2024-12-29 21:29:25,518 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 12 +2024-12-29 21:29:25,518 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:25,518 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-29 21:29:25,518 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:25,519 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-29 21:29:25,519 INFO SenderThread:1360479 [file_stream.py:finish():601] file stream finish called +2024-12-29 21:29:25,708 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:29:25,725 INFO SenderThread:1360479 [file_stream.py:finish():605] file stream finish is done +2024-12-29 21:29:25,725 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 13 +2024-12-29 21:29:25,725 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:29:25,725 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:25,725 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-29 21:29:25,725 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:25,725 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-29 21:29:25,725 INFO SenderThread:1360479 [sender.py:transition_state():614] send defer: 14 +2024-12-29 21:29:25,726 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: defer +2024-12-29 21:29:25,726 DEBUG SenderThread:1360479 [sender.py:send():379] send: final +2024-12-29 21:29:25,726 INFO HandlerThread:1360479 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-29 21:29:25,726 DEBUG SenderThread:1360479 [sender.py:send():379] send: footer +2024-12-29 21:29:25,726 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: defer +2024-12-29 21:29:25,726 INFO SenderThread:1360479 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-29 21:29:25,726 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:29:25,726 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:29:25,727 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: server_info +2024-12-29 21:29:25,727 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: server_info +2024-12-29 21:29:25,728 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 21:29:25,728 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: get_summary +2024-12-29 21:29:25,729 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-29 21:29:25,729 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 21:29:25,782 DEBUG SenderThread:1360479 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 21:29:25,783 INFO MainThread:1360479 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-29 21:29:25,783 INFO MainThread:1360479 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-29 21:29:25,783 INFO MainThread:1360479 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-29 21:29:25,783 DEBUG HandlerThread:1360479 [handler.py:handle_request():158] handle_request: shutdown +2024-12-29 21:29:25,783 INFO HandlerThread:1360479 [handler.py:finish():882] shutting down handler +2024-12-29 21:29:26,728 INFO WriterThread:1360479 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/run-oco0rjll.wandb +2024-12-29 21:29:26,783 INFO SenderThread:1360479 [sender.py:finish():1615] shutting down sender +2024-12-29 21:29:26,783 INFO SenderThread:1360479 [file_pusher.py:finish():169] shutting down file pusher +2024-12-29 21:29:26,783 INFO SenderThread:1360479 [file_pusher.py:join():175] waiting for file pusher diff --git a/212506/wandb/run-20241229_212512-oco0rjll/logs/debug.log b/212506/wandb/run-20241229_212512-oco0rjll/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f58fcb13bfaf67c74135f2592ae4895b10943301 --- /dev/null +++ b/212506/wandb/run-20241229_212512-oco0rjll/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-29 21:25:12,623 INFO MainThread:1360290 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_setup.py:_flush():76] Configure stats pid to 1360290 +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/settings +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/logs/debug.log +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/wandb/run-20241229_212512-oco0rjll/logs/debug-internal.log +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_init.py:init():569] calling init triggers +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1000_0', 'wandb_notes': '1000_0_req_1351_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506'} +2024-12-29 21:25:12,624 INFO MainThread:1360290 [wandb_init.py:init():619] starting backend +2024-12-29 21:25:12,625 INFO MainThread:1360290 [wandb_init.py:init():623] setting up manager +2024-12-29 21:25:12,628 INFO MainThread:1360290 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-29 21:25:12,629 INFO MainThread:1360290 [wandb_init.py:init():631] backend started and connected +2024-12-29 21:25:12,640 INFO MainThread:1360290 [wandb_init.py:init():720] updated telemetry +2024-12-29 21:25:12,649 INFO MainThread:1360290 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-29 21:25:13,005 INFO MainThread:1360290 [wandb_run.py:_on_init():2435] communicating current version +2024-12-29 21:25:13,082 INFO MainThread:1360290 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-29 21:25:13,082 INFO MainThread:1360290 [wandb_init.py:init():804] starting run threads in backend +2024-12-29 21:25:13,419 INFO MainThread:1360290 [wandb_run.py:_console_start():2413] atexit reg +2024-12-29 21:25:13,419 INFO MainThread:1360290 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-29 21:25:13,419 INFO MainThread:1360290 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-29 21:25:13,420 INFO MainThread:1360290 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-29 21:25:13,421 INFO MainThread:1360290 [wandb_init.py:init():847] run started, returning control to user process +2024-12-29 21:25:13,422 INFO MainThread:1360290 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1000_0/212506/tb, True +2024-12-29 21:25:17,662 INFO MainThread:1360290 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2024-12-29 21:29:26,788 WARNING MsgRouterThr:1360290 [router.py:message_loop():77] message_loop has been closed diff --git a/212506/wandb/run-20241229_212512-oco0rjll/run-oco0rjll.wandb b/212506/wandb/run-20241229_212512-oco0rjll/run-oco0rjll.wandb new file mode 100644 index 0000000000000000000000000000000000000000..a537a3d3e7b9643d5749f3a543f024ed15508ab6 --- /dev/null +++ b/212506/wandb/run-20241229_212512-oco0rjll/run-oco0rjll.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4e7d87f4a9a568f42e21166a49e77e36045dcf45ec0536bedcb54c292ec1daa +size 282015 diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cae4a264b194787d3b0781195ecfaa3104dab6c5 --- /dev/null +++ b/config.yaml @@ -0,0 +1,546 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 0 +dataset_shuffle_seed: 0 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '1000_0' +wandb_notes: 1000_0_req_1351_0restarted_2 +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: all +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 0 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/snapshot_10500.pt b/snapshot_10500.pt new file mode 100644 index 0000000000000000000000000000000000000000..f01a1ce6b8ce8b90fa9b57b2314914a91e4e4e32 --- /dev/null +++ b/snapshot_10500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa0770fd2cb8be049f6611b019974aec2bb2b7fe7678d6bfe88ad1ee5447bef2 +size 910102870 diff --git a/snapshot_10999.pt b/snapshot_10999.pt new file mode 100644 index 0000000000000000000000000000000000000000..98845e868faeaa443bda19e637dbaf1a7b14a1aa --- /dev/null +++ b/snapshot_10999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99ce2b49f4e0e75a57bda5210acc3afe0eb45c9b249d62cc0aa33a8283d4f6f4 +size 910102870 diff --git a/snapshot_11249.pt b/snapshot_11249.pt new file mode 100644 index 0000000000000000000000000000000000000000..49a14ef3a155462b0d060f47067790db38b1c2e2 --- /dev/null +++ b/snapshot_11249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a2a671f4894e792c623f385fe7f4e4778201617b83359bcaae21a3883f27016 +size 910102870 diff --git a/snapshot_11499.pt b/snapshot_11499.pt new file mode 100644 index 0000000000000000000000000000000000000000..9605d57fcfa84df71efd5ec343964da37d2bb68c --- /dev/null +++ b/snapshot_11499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d11f1dfabe263e7b58fb9bf21b98c01be444fee75403a93b6d146df800381f7e +size 910102870 diff --git a/snapshot_11749.pt b/snapshot_11749.pt new file mode 100644 index 0000000000000000000000000000000000000000..999b00e4bd651e5b3cf2d002fc16b6f0115bfdc3 --- /dev/null +++ b/snapshot_11749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8004793f91f54dad4864acbffa9fc265998e4ad1078a9a7827cda955ccfc7691 +size 910102870 diff --git a/snapshot_11999.pt b/snapshot_11999.pt new file mode 100644 index 0000000000000000000000000000000000000000..6d9e0d60e784a00e71a1d3750b366f5586031857 --- /dev/null +++ b/snapshot_11999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f19206e897f5aa706e4d0f28d12f8158642a42d05cc3ea4bc86c76aa389bc33 +size 910102870 diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..7a373093457c5a236aec4abb07d4a1a630ffde8a --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bca6c2b52d9837b9cea05dfccae4c0e0f143a5972269755579163b7ad0c03e4 +size 910102870 diff --git a/snapshot_12208.pt b/snapshot_12208.pt new file mode 100644 index 0000000000000000000000000000000000000000..c7e6d12882e2dabace96a6d37c791d17f58fd9db --- /dev/null +++ b/snapshot_12208.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d592fad964da8a0a6529d9502a2224c366bf237dda36e4e60116732b45ca2a6a +size 910102870 diff --git a/snapshot_12999.pt b/snapshot_12999.pt new file mode 100644 index 0000000000000000000000000000000000000000..f69424f496416c299add8903cdf0da62406a42c8 --- /dev/null +++ b/snapshot_12999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83d70f23e9d1877655f53e51405245ccce77a10e2d9e8ea46ca0526c851cbae7 +size 910102870 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..82e7aa29edfba4ebec341dad5d1e633db7a03947 --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe82710f85de9eee1cc19030716fd56faacd9fb8b36a02c7f012ce03f5b24364 +size 910102870 diff --git a/snapshot_13749.pt b/snapshot_13749.pt new file mode 100644 index 0000000000000000000000000000000000000000..c67fc81d2f92a41aa6d32552de59961917bdd6ed --- /dev/null +++ b/snapshot_13749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f5b4133eb0f13e2e7475fbf0d44ed8e1c9a4820e78924ca1f79b26c7a71314a +size 910102870 diff --git a/snapshot_13999.pt b/snapshot_13999.pt new file mode 100644 index 0000000000000000000000000000000000000000..48a318f2d729563ede67af1aa137df33c26ddcd2 --- /dev/null +++ b/snapshot_13999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89bd94d874c7662396515218a32bbfbf8d9f42f4bfe4c6acc449029a3ae0c083 +size 910102870 diff --git a/snapshot_14499.pt b/snapshot_14499.pt new file mode 100644 index 0000000000000000000000000000000000000000..937f84a43df32bc066ba775b045f472347404d9e --- /dev/null +++ b/snapshot_14499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26751983a106f11ecfafff48ac107dc7ed60dd154c0e236328ca5896c5b110f9 +size 910102870 diff --git a/snapshot_14749.pt b/snapshot_14749.pt new file mode 100644 index 0000000000000000000000000000000000000000..809f0c4538a87d152f4cd0f63a169f773f25d871 --- /dev/null +++ b/snapshot_14749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41647c371b98423396d3366b189dbf059a7a1a80c4db2025565a0a6267c7db45 +size 910102870 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..5a55e1b3195b6fcc59d59f66c17ff6ee55e5a0ca --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44ac27481073d231cd3bc1cf1e309fc3a1b327ebe0d0c3acfc942981b63e6406 +size 910102870 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f2e7fcc668d0d13cba60ce253fc871dd66e09e7 --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1227ba067af5c9a21b959b6d9034376efbb9f25a0268e35429ec5336eaefe7a6 +size 910102870 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd0653d49bb5d8cf930200f73a18b93fab99c413 --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86049954860adeb694b445648e12dd3aee4607bc9691c5a1492863d740c6cb38 +size 910102870 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..debd1163e9322f18657828a98e9232fba223aece --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30ca1b1ed099ae3d7c56e9b3d2d2b9a687a82e5e486ec2394671df389d7ab16b +size 910102870 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..6224da90833da6d7ed34af8d3834860c5acd6de4 --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e04c3d5bb5b5f2dc6b98d209b83929dfed0f6c650601412e29e4e197cab68a9 +size 910102870 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..5a55e1b3195b6fcc59d59f66c17ff6ee55e5a0ca --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44ac27481073d231cd3bc1cf1e309fc3a1b327ebe0d0c3acfc942981b63e6406 +size 910102870