diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..501065943f6457b35308b462ea63eaac71393fe0 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,19 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +113638/wandb/run-20241211_113644-ahosb0lo/run-ahosb0lo.wandb filter=lfs diff=lfs merge=lfs -text +113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/2_eval_2_ddba0e3583786ac1097b.mp4 filter=lfs diff=lfs merge=lfs -text +113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/3_eval_3_b687720b2d32aeeb9a91.mp4 filter=lfs diff=lfs merge=lfs -text +113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/4_eval_4_10441113b4d3ea44e8c6.mp4 filter=lfs diff=lfs merge=lfs -text +113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/0_eval_0_dea3cb7ce16ad057bd5f.mp4 filter=lfs diff=lfs merge=lfs -text +113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/1_eval_1_925210ff3b36d88f3887.mp4 filter=lfs diff=lfs merge=lfs -text +113638/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +113638/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +113638/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +113638/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +113638/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +113638/episode_rosbags/episode_3_2024-12-11-11-40-13.bag filter=lfs diff=lfs merge=lfs -text +113638/episode_rosbags/episode_4_2024-12-11-11-41-04.bag filter=lfs diff=lfs merge=lfs -text +113638/episode_rosbags/episode_2_2024-12-11-11-39-24.bag filter=lfs diff=lfs merge=lfs -text +113638/episode_rosbags/episode_1_2024-12-11-11-38-11.bag filter=lfs diff=lfs merge=lfs -text +113638/episode_rosbags/episode_0_2024-12-11-11-37-21.bag filter=lfs diff=lfs merge=lfs -text diff --git a/113637/.hydra/config.yaml b/113637/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..28fe84c2c9470abdf3e6696c99f65fffd3c368b1 --- /dev/null +++ b/113637/.hydra/config.yaml @@ -0,0 +1,349 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '265650' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/113637/.hydra/hydra.yaml b/113637/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f6b7f4be69abda4afa9a5d48fe7195059ddd8f56 --- /dev/null +++ b/113637/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${experiment_dir}/${now:%H%M%S} + sweep: + dir: ${experiment_dir}/${now:%H%M%S} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${experiment_dir}/${now:%H%M%S}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113637 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/113637/.hydra/overrides.yaml b/113637/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/113637/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/113637/eval_policy.log b/113637/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..92e2b46e804ced21e918b54922e5aa57be737c24 --- /dev/null +++ b/113637/eval_policy.log @@ -0,0 +1,15 @@ +[2024-12-11 11:36:37,909][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-11 11:36:37,913][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2024-12-11 11:36:40,078][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/113638/.hydra/config.yaml b/113638/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..28fe84c2c9470abdf3e6696c99f65fffd3c368b1 --- /dev/null +++ b/113638/.hydra/config.yaml @@ -0,0 +1,349 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '265650' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/113638/.hydra/hydra.yaml b/113638/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dc39a90beddd0b8250b00adb9ac19a5edff8b14e --- /dev/null +++ b/113638/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${experiment_dir}/${now:%H%M%S} + sweep: + dir: ${experiment_dir}/${now:%H%M%S} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${experiment_dir}/${now:%H%M%S}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/113638/.hydra/overrides.yaml b/113638/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/113638/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/113638/episode_rosbags/aligned_depth_to_color_K.npy b/113638/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/113638/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/113638/episode_rosbags/cam_tf_world.npy b/113638/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/113638/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/113638/episode_rosbags/color_K.npy b/113638/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/113638/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/113638/episode_rosbags/depth_K.npy b/113638/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/113638/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/113638/episode_rosbags/episode_0_2024-12-11-11-37-21.bag b/113638/episode_rosbags/episode_0_2024-12-11-11-37-21.bag new file mode 100644 index 0000000000000000000000000000000000000000..8a1b9e3fafc6fcfb2a8f482a47548f612dd405f7 --- /dev/null +++ b/113638/episode_rosbags/episode_0_2024-12-11-11-37-21.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74bc90c7db93c607b967f9bc030866df3e80592ea3e65471c2351a0175390234 +size 2623667119 diff --git a/113638/episode_rosbags/episode_1_2024-12-11-11-38-11.bag b/113638/episode_rosbags/episode_1_2024-12-11-11-38-11.bag new file mode 100644 index 0000000000000000000000000000000000000000..aef43c16e04554d078fd7d804544772c3ca86310 --- /dev/null +++ b/113638/episode_rosbags/episode_1_2024-12-11-11-38-11.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daa007fc960ef8b8f8b1b4dcdc7952b67485ebddb405bebb85a8573ec8f10c07 +size 2597056688 diff --git a/113638/episode_rosbags/episode_2_2024-12-11-11-39-24.bag b/113638/episode_rosbags/episode_2_2024-12-11-11-39-24.bag new file mode 100644 index 0000000000000000000000000000000000000000..399b7e259225983068fc7aa12536a21eed91bb84 --- /dev/null +++ b/113638/episode_rosbags/episode_2_2024-12-11-11-39-24.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:375221ca835c5bfea3c361551cc0ffbabc20af5b4420093df8c9924b1607f57e +size 2619130394 diff --git a/113638/episode_rosbags/episode_3_2024-12-11-11-40-13.bag b/113638/episode_rosbags/episode_3_2024-12-11-11-40-13.bag new file mode 100644 index 0000000000000000000000000000000000000000..33e50b710a789c29b013f6ba7e7b27700f69ace7 --- /dev/null +++ b/113638/episode_rosbags/episode_3_2024-12-11-11-40-13.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd05f0d20454eac597268244d15aa5b7cdfcdc95ff8813e0bab97598d552509e +size 2614967649 diff --git a/113638/episode_rosbags/episode_4_2024-12-11-11-41-04.bag b/113638/episode_rosbags/episode_4_2024-12-11-11-41-04.bag new file mode 100644 index 0000000000000000000000000000000000000000..073e2cc0cc30f3ebcd6d88d2c5fbb86d3ea3fc8f --- /dev/null +++ b/113638/episode_rosbags/episode_4_2024-12-11-11-41-04.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ff38aff071b614754716e52ff85c66afa8284cf8a48965237df3ae8a42f1220 +size 2616371914 diff --git a/113638/eval_robot.log b/113638/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..87056edf5289562911c4f744a69c3ffd70ed3237 --- /dev/null +++ b/113638/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-11 11:36:38,723][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-11 11:36:38,726][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/113638/eval_video/0_eval.mp4 b/113638/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6752b9c0bed742a62ee0ef6418806fd2c76c1b4e --- /dev/null +++ b/113638/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dea3cb7ce16ad057bd5fcb35b50592a86feeb9f7d6f3bd7b6f42095d27997931 +size 1422243 diff --git a/113638/eval_video/1_eval.mp4 b/113638/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..93c254d4f258ecfa7174a6a0bccd93ad07680a83 --- /dev/null +++ b/113638/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:925210ff3b36d88f3887c6bf561b993eeb2172f9b898edf7efaa91f7c720e590 +size 1468875 diff --git a/113638/eval_video/2_eval.mp4 b/113638/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a4b782d08953681dc476750407ad8b71b6e1ec08 --- /dev/null +++ b/113638/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddba0e3583786ac1097bd8d918d5801eb4867cac0e908be1695101412c576aec +size 1498755 diff --git a/113638/eval_video/3_eval.mp4 b/113638/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a5831c1b674457255f4f48885d3fde460c528a80 --- /dev/null +++ b/113638/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b687720b2d32aeeb9a91a18ae8924160e7860fcc22b2850d7f2b3b59ca27c2ec +size 1454836 diff --git a/113638/eval_video/4_eval.mp4 b/113638/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7ce87e470e0e1a9f82045e8a1686732a7fe0cfa3 --- /dev/null +++ b/113638/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10441113b4d3ea44e8c6df553d629edb8882f588fa5814fc108ba5370ac0ec0d +size 1451627 diff --git a/113638/tb/events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 b/113638/tb/events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 new file mode 100644 index 0000000000000000000000000000000000000000..8263e23a8d469f5a1404101aa2a3b406d378a09e --- /dev/null +++ b/113638/tb/events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc2777e686dca07c00a1f3ef6536f0fb088e018afe2d03ca0d9b75f1f5570f46 +size 1103 diff --git a/113638/wandb/debug-internal.log b/113638/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c56fb179ccad396ea87cb0c61ff4691bc6f0f4d3 --- /dev/null +++ b/113638/wandb/debug-internal.log @@ -0,0 +1,840 @@ +2024-12-11 11:36:44,620 INFO StreamThr :743447 [internal.py:wandb_internal():85] W&B internal server running at pid: 743447, started at: 2024-12-11 11:36:44.620258 +2024-12-11 11:36:44,621 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status +2024-12-11 11:36:44,632 INFO WriterThread:743447 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/run-ahosb0lo.wandb +2024-12-11 11:36:44,633 DEBUG SenderThread:743447 [sender.py:send():379] send: header +2024-12-11 11:36:44,642 DEBUG SenderThread:743447 [sender.py:send():379] send: run +2024-12-11 11:36:44,901 INFO SenderThread:743447 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files +2024-12-11 11:36:44,901 INFO SenderThread:743447 [sender.py:_start_run_threads():1188] run started: ahosb0lo with start time 1733935004.620625 +2024-12-11 11:36:44,921 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: check_version +2024-12-11 11:36:44,921 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: check_version +2024-12-11 11:36:44,995 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: run_start +2024-12-11 11:36:45,000 DEBUG HandlerThread:743447 [system_info.py:__init__():26] System info init +2024-12-11 11:36:45,000 DEBUG HandlerThread:743447 [system_info.py:__init__():41] System info init done +2024-12-11 11:36:45,000 INFO HandlerThread:743447 [system_monitor.py:start():194] Starting system monitor +2024-12-11 11:36:45,000 INFO SystemMonitor:743447 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-11 11:36:45,000 INFO HandlerThread:743447 [system_monitor.py:probe():214] Collecting system info +2024-12-11 11:36:45,000 INFO SystemMonitor:743447 [interfaces.py:start():188] Started cpu monitoring +2024-12-11 11:36:45,001 INFO SystemMonitor:743447 [interfaces.py:start():188] Started disk monitoring +2024-12-11 11:36:45,002 INFO SystemMonitor:743447 [interfaces.py:start():188] Started gpu monitoring +2024-12-11 11:36:45,002 INFO SystemMonitor:743447 [interfaces.py:start():188] Started memory monitoring +2024-12-11 11:36:45,003 INFO SystemMonitor:743447 [interfaces.py:start():188] Started network monitoring +2024-12-11 11:36:45,012 DEBUG HandlerThread:743447 [system_info.py:probe():152] Probing system +2024-12-11 11:36:45,013 DEBUG HandlerThread:743447 [system_info.py:_probe_git():137] Probing git +2024-12-11 11:36:45,018 DEBUG HandlerThread:743447 [system_info.py:_probe_git():145] Probing git done +2024-12-11 11:36:45,018 DEBUG HandlerThread:743447 [system_info.py:probe():200] Probing system done +2024-12-11 11:36:45,018 DEBUG HandlerThread:743447 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-11T16:36:45.012114', 'startedAt': '2024-12-11T16:36:44.613038', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '1b557f164c7f62a8d1545c761b13ede1677c3a83'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.48675, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.011, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 661.6516265869141}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71596908569336}} +2024-12-11 11:36:45,018 INFO HandlerThread:743447 [system_monitor.py:probe():224] Finished collecting system info +2024-12-11 11:36:45,018 INFO HandlerThread:743447 [system_monitor.py:probe():227] Publishing system info +2024-12-11 11:36:45,018 DEBUG HandlerThread:743447 [system_info.py:_save_code():44] Saving code +2024-12-11 11:36:45,020 DEBUG HandlerThread:743447 [system_info.py:_save_code():67] Saving code done +2024-12-11 11:36:45,020 DEBUG HandlerThread:743447 [system_info.py:_save_patches():84] Saving git patches +2024-12-11 11:36:45,082 DEBUG HandlerThread:743447 [system_info.py:_save_patches():126] Saving git patches done +2024-12-11 11:36:45,082 INFO HandlerThread:743447 [system_monitor.py:probe():229] Finished publishing system info +2024-12-11 11:36:45,086 DEBUG SenderThread:743447 [sender.py:send():379] send: files +2024-12-11 11:36:45,086 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-11 11:36:45,086 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-11 11:36:45,086 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-11 11:36:45,328 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: python_packages +2024-12-11 11:36:45,328 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:36:45,329 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: python_packages +2024-12-11 11:36:45,329 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:36:45,330 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:45,331 INFO HandlerThread:743447 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/tb" + save: true +} +control { + relay_id: "3b814f0f1e304546b568a989da857bea" +} +_info { + stream_id: "ahosb0lo" +} + +2024-12-11 11:36:45,331 DEBUG HandlerThread:743447 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-11 11:36:45,344 INFO wandb-upload_0:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/jpxgwjga-wandb-metadata.json +2024-12-11 11:36:45,381 INFO wandb-upload_1:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/xzmf7vax-code/FISH/eval_robot.py +2024-12-11 11:36:45,383 INFO wandb-upload_2:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/am9eq8ny-diff.patch +2024-12-11 11:36:45,439 DEBUG SenderThread:743447 [sender.py:send():379] send: telemetry +2024-12-11 11:36:45,439 DEBUG SenderThread:743447 [sender.py:send():379] send: tbrecord +2024-12-11 11:36:45,439 DEBUG SenderThread:743447 [sender.py:send():379] send: files +2024-12-11 11:36:45,439 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 with policy live +2024-12-11 11:36:45,614 INFO wandb-upload_0:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/sdv9591m-events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:36:45,902 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-metadata.json +2024-12-11 11:36:45,902 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/diff.patch +2024-12-11 11:36:45,903 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:36:45,903 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/requirements.txt +2024-12-11 11:36:45,903 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:36:45,903 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/code/FISH/eval_robot.py +2024-12-11 11:36:45,903 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/code/FISH +2024-12-11 11:36:45,903 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/code +2024-12-11 11:36:46,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:47,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:47,902 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:36:48,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:49,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:49,565 DEBUG SenderThread:743447 [sender.py:send():379] send: config +2024-12-11 11:36:50,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:36:50,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:51,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:51,903 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:36:52,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:53,330 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:53,903 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:36:54,346 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:55,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:36:55,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:55,904 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:36:56,330 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:57,330 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:57,905 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:36:58,330 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:59,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:00,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:37:00,329 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:37:00,370 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:00,483 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:01,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:02,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:03,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:04,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:05,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:06,330 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:06,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:07,332 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:08,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:09,332 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:10,332 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:11,330 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:11,332 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:12,332 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:13,332 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:14,332 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:15,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:37:15,329 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:37:15,370 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:16,333 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:16,419 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:16,910 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/config.yaml +2024-12-11 11:37:17,333 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:18,333 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:19,333 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:19,911 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:20,333 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:21,333 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:21,687 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:22,334 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:23,334 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:23,912 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:24,334 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:25,334 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:25,912 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:26,334 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:26,799 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:27,336 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:27,913 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:28,336 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:29,337 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:29,913 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:30,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:37:30,329 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:37:30,337 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:31,337 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:31,803 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:31,914 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:32,337 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:33,338 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:33,914 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:34,337 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:35,338 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:35,915 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:36,338 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:37,047 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:37,338 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:37,916 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:38,339 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:39,339 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:39,916 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:40,339 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:41,339 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:41,917 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:42,166 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:42,339 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:43,340 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:43,918 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:44,340 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:45,003 DEBUG SystemMonitor:743447 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-11 11:37:45,004 DEBUG SenderThread:743447 [sender.py:send():379] send: stats +2024-12-11 11:37:45,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:37:45,329 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:37:45,340 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:45,918 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:46,340 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:47,340 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:47,422 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:47,919 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:48,340 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:49,340 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:49,919 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:50,340 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:51,341 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:51,920 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:52,341 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:52,603 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:53,341 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:53,921 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:54,341 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:55,341 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:55,921 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:56,341 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:57,288 DEBUG SenderThread:743447 [sender.py:send():379] send: files +2024-12-11 11:37:57,289 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:37:57,289 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_dea3cb7ce16ad057bd5f.mp4 with policy now +2024-12-11 11:37:57,290 DEBUG SenderThread:743447 [sender.py:send():379] send: history +2024-12-11 11:37:57,290 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:37:57,290 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:37:57,342 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:57,529 INFO wandb-upload_2:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/s4xw9bmh-media/videos/eval/0_eval_0_dea3cb7ce16ad057bd5f.mp4 +2024-12-11 11:37:57,922 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:57,922 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/0_eval_0_dea3cb7ce16ad057bd5f.mp4 +2024-12-11 11:37:57,923 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json +2024-12-11 11:37:57,923 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos +2024-12-11 11:37:57,923 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval +2024-12-11 11:37:57,923 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media +2024-12-11 11:37:58,202 INFO wandb-upload_0:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/7xrrwz3m-media/videos/eval/0_eval_0_dea3cb7ce16ad057bd5f.mp4 +2024-12-11 11:37:58,302 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:58,342 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:59,342 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:59,923 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:00,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:38:00,329 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:38:00,342 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:01,342 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:01,923 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:02,342 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:03,345 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:03,924 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:03,962 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:38:04,344 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:05,344 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:06,344 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:07,344 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:07,925 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:38:08,171 INFO wandb-upload_2:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/pvc3l7fp-events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:38:08,345 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:09,345 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:09,794 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:38:09,926 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:10,345 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:11,346 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:12,345 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:13,346 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:13,927 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:14,348 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:14,851 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:38:15,004 DEBUG SenderThread:743447 [sender.py:send():379] send: stats +2024-12-11 11:38:15,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:38:15,330 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:38:15,346 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:15,928 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:16,346 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:17,347 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:17,929 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:18,347 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:19,347 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:19,929 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:20,073 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:38:20,347 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:21,348 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:21,930 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:22,347 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:23,347 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:23,931 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:24,347 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:25,115 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:38:25,347 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:25,931 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:26,347 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:27,348 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:27,932 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:28,348 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:29,348 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:29,933 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:30,147 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:38:30,330 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:38:30,330 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:38:30,349 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:31,348 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:31,934 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:32,349 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:33,348 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:33,934 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:34,348 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:35,171 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:38:35,349 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:35,935 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:36,350 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:37,350 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:37,936 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:38,350 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:39,350 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:39,936 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:40,215 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:38:40,352 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:41,353 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:41,937 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:42,354 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:43,354 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:43,938 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:44,354 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:45,004 DEBUG SenderThread:743447 [sender.py:send():379] send: stats +2024-12-11 11:38:45,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:38:45,331 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:38:45,354 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:45,447 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:38:45,938 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:46,354 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:47,357 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:47,749 DEBUG SenderThread:743447 [sender.py:send():379] send: files +2024-12-11 11:38:47,749 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_925210ff3b36d88f3887.mp4 with policy now +2024-12-11 11:38:47,750 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:38:47,751 DEBUG SenderThread:743447 [sender.py:send():379] send: history +2024-12-11 11:38:47,751 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:38:47,759 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:38:47,939 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:47,939 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json +2024-12-11 11:38:47,940 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/1_eval_1_925210ff3b36d88f3887.mp4 +2024-12-11 11:38:47,940 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval +2024-12-11 11:38:47,988 INFO wandb-upload_0:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/algrlwjl-media/videos/eval/1_eval_1_925210ff3b36d88f3887.mp4 +2024-12-11 11:38:48,259 INFO wandb-upload_2:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/eex9yxjq-media/videos/eval/1_eval_1_925210ff3b36d88f3887.mp4 +2024-12-11 11:38:48,365 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:49,358 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:49,940 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:50,357 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:50,450 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:38:51,358 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:51,941 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:52,363 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:53,358 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:53,941 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:54,361 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:55,357 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:56,336 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:38:56,357 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:57,358 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:58,358 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:59,364 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:00,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:39:00,331 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:39:00,377 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:01,364 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:01,423 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:02,364 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:03,367 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:04,366 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:05,375 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:06,369 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:07,338 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:07,370 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:08,367 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:09,369 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:10,371 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:11,376 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:12,343 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:12,373 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:13,372 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:14,371 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:15,005 DEBUG SenderThread:743447 [sender.py:send():379] send: stats +2024-12-11 11:39:15,335 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:39:15,335 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:39:15,377 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:16,373 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:17,372 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:17,430 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:18,373 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:19,374 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:20,373 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:21,373 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:21,758 DEBUG SenderThread:743447 [sender.py:send():379] send: history +2024-12-11 11:39:21,759 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:39:21,759 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:39:21,950 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:39:21,950 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json +2024-12-11 11:39:21,951 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:22,200 INFO wandb-upload_0:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/wnjvdsbf-events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:39:22,374 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:23,170 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:23,373 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:23,951 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:24,374 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:25,374 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:25,951 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:26,373 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:27,373 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:27,952 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:28,298 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:28,375 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:29,375 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:29,953 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:30,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:39:30,332 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:39:30,374 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:31,374 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:31,954 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:32,375 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:33,322 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:33,375 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:33,954 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:34,376 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:35,375 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:35,955 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:36,375 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:37,376 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:37,956 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:38,344 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:38,376 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:39,377 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:39,957 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:40,376 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:41,376 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:41,958 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:42,376 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:43,376 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:43,454 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:43,958 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:44,376 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:45,005 DEBUG SenderThread:743447 [sender.py:send():379] send: stats +2024-12-11 11:39:45,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:39:45,332 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:39:45,377 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:45,959 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:46,377 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:47,377 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:47,960 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:48,377 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:48,576 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:49,377 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:49,961 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:50,377 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:51,378 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:51,961 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:52,378 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:53,378 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:53,803 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:53,962 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:54,378 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:55,378 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:55,963 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:56,378 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:57,378 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:57,964 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:58,378 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:58,900 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:59,378 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:59,964 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:00,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:40:00,332 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:40:00,378 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:01,379 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:01,965 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:02,384 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:02,517 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:40:02,518 DEBUG SenderThread:743447 [sender.py:send():379] send: files +2024-12-11 11:40:02,518 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_ddba0e3583786ac1097b.mp4 with policy now +2024-12-11 11:40:02,519 DEBUG SenderThread:743447 [sender.py:send():379] send: history +2024-12-11 11:40:02,519 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:40:02,519 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:40:02,735 INFO wandb-upload_2:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/2qt74j26-media/videos/eval/2_eval_2_ddba0e3583786ac1097b.mp4 +2024-12-11 11:40:02,965 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json +2024-12-11 11:40:02,966 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/2_eval_2_ddba0e3583786ac1097b.mp4 +2024-12-11 11:40:02,966 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval +2024-12-11 11:40:03,258 INFO wandb-upload_0:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/nmnl78ir-media/videos/eval/2_eval_2_ddba0e3583786ac1097b.mp4 +2024-12-11 11:40:03,390 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:03,966 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:04,387 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:04,529 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:40:05,387 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:05,966 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:06,386 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:07,389 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:08,386 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:09,394 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:09,533 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:40:09,621 DEBUG SenderThread:743447 [sender.py:send():379] send: history +2024-12-11 11:40:09,622 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:40:09,622 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:40:09,968 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:40:09,968 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json +2024-12-11 11:40:09,968 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:10,392 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:11,387 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:11,968 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:12,398 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:13,391 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:13,969 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:14,389 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:14,562 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:40:15,006 DEBUG SenderThread:743447 [sender.py:send():379] send: stats +2024-12-11 11:40:15,332 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:40:15,332 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:40:15,389 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:15,970 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:16,389 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:17,389 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:17,971 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:18,389 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:19,389 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:19,788 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:40:19,972 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:20,389 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:21,389 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:21,973 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:22,389 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:23,390 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:23,974 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:24,390 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:24,792 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:40:25,390 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:25,974 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:26,390 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:27,390 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:27,975 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:28,390 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:29,391 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:29,860 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:40:29,976 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:30,332 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:40:30,332 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:40:30,391 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:31,391 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:31,977 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:32,391 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:33,391 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:33,977 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:34,392 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:35,029 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:40:35,392 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:35,978 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:36,392 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:37,392 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:37,979 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:38,392 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:39,392 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:39,979 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:40,086 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:40:40,392 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:41,393 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:41,980 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:42,393 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:43,393 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:43,981 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:44,393 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:45,014 DEBUG SenderThread:743447 [sender.py:send():379] send: stats +2024-12-11 11:40:45,130 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:40:45,333 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:40:45,333 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:40:45,393 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:45,982 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:46,393 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:47,393 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:47,983 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:48,393 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:49,396 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:49,983 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:50,091 DEBUG SenderThread:743447 [sender.py:send():379] send: files +2024-12-11 11:40:50,091 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_b687720b2d32aeeb9a91.mp4 with policy now +2024-12-11 11:40:50,094 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:40:50,094 DEBUG SenderThread:743447 [sender.py:send():379] send: history +2024-12-11 11:40:50,095 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:40:50,105 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:40:50,133 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:40:50,329 INFO wandb-upload_2:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/6p2eraxb-media/videos/eval/3_eval_3_b687720b2d32aeeb9a91.mp4 +2024-12-11 11:40:50,394 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:50,987 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json +2024-12-11 11:40:50,987 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/3_eval_3_b687720b2d32aeeb9a91.mp4 +2024-12-11 11:40:50,987 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval +2024-12-11 11:40:51,327 INFO wandb-upload_0:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/aiy14acg-media/videos/eval/3_eval_3_b687720b2d32aeeb9a91.mp4 +2024-12-11 11:40:51,406 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:51,987 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:52,399 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:53,404 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:53,988 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:54,403 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:55,358 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:40:55,409 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:56,411 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:57,413 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:57,989 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:58,410 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:59,410 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:00,335 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:41:00,336 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:41:00,411 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:00,432 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:00,683 DEBUG SenderThread:743447 [sender.py:send():379] send: history +2024-12-11 11:41:00,683 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:41:00,683 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:41:00,991 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:41:00,991 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json +2024-12-11 11:41:01,267 INFO wandb-upload_2:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/8cztmbl3-events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:41:01,410 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:01,991 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:02,412 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:03,414 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:04,409 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:05,446 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:05,446 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:05,992 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:06,421 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:07,421 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:07,993 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:08,422 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:09,422 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:09,994 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:10,422 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:10,633 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:11,422 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:11,994 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:12,422 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:13,422 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:13,995 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:14,422 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:15,007 DEBUG SenderThread:743447 [sender.py:send():379] send: stats +2024-12-11 11:41:15,333 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:41:15,334 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:41:15,423 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:15,812 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:15,996 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:16,423 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:17,424 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:17,997 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:18,425 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:19,426 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:19,998 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:20,425 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:20,844 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:21,425 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:21,998 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:22,425 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:23,425 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:23,999 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:24,425 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:25,425 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:25,874 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:26,000 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:26,425 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:27,426 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:28,001 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:28,431 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:29,430 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:30,001 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:30,333 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:41:30,334 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:41:30,430 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:31,070 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:31,430 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:32,002 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:32,430 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:33,430 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:34,003 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:34,431 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:35,431 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:36,004 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:36,123 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:36,431 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:37,431 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:38,005 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:38,431 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:39,431 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:40,005 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:40,433 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:40,811 DEBUG SenderThread:743447 [sender.py:send():379] send: files +2024-12-11 11:41:40,811 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_10441113b4d3ea44e8c6.mp4 with policy now +2024-12-11 11:41:40,812 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:41:40,812 DEBUG SenderThread:743447 [sender.py:send():379] send: history +2024-12-11 11:41:40,812 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:41:40,820 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:41:41,006 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json +2024-12-11 11:41:41,006 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/4_eval_4_10441113b4d3ea44e8c6.mp4 +2024-12-11 11:41:41,006 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval +2024-12-11 11:41:41,043 INFO wandb-upload_0:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/0somhq86-media/videos/eval/4_eval_4_10441113b4d3ea44e8c6.mp4 +2024-12-11 11:41:41,317 INFO wandb-upload_2:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/56mpvpsh-media/videos/eval/4_eval_4_10441113b4d3ea44e8c6.mp4 +2024-12-11 11:41:41,437 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:41,867 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:42,006 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:42,441 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:43,435 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:44,007 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:44,439 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:45,007 DEBUG SenderThread:743447 [sender.py:send():379] send: stats +2024-12-11 11:41:45,345 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:41:45,345 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:41:45,442 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:46,441 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:47,446 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:47,542 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:48,009 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:48,446 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:49,450 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:50,455 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:51,330 DEBUG SenderThread:743447 [sender.py:send():379] send: exit +2024-12-11 11:41:51,330 INFO SenderThread:743447 [sender.py:send_exit():586] handling exit code: 0 +2024-12-11 11:41:51,330 INFO SenderThread:743447 [sender.py:send_exit():588] handling runtime: 306 +2024-12-11 11:41:51,330 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:41:51,330 INFO SenderThread:743447 [sender.py:send_exit():594] send defer +2024-12-11 11:41:51,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:41:51,331 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-11 11:41:51,331 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:41:51,331 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-11 11:41:51,331 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 1 +2024-12-11 11:41:51,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:41:51,331 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-11 11:41:51,331 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:41:51,331 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-11 11:41:51,331 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 2 +2024-12-11 11:41:51,332 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:41:51,332 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-11 11:41:51,332 INFO HandlerThread:743447 [system_monitor.py:finish():203] Stopping system monitor +2024-12-11 11:41:51,332 DEBUG SystemMonitor:743447 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-11 11:41:51,332 INFO HandlerThread:743447 [interfaces.py:finish():200] Joined cpu monitor +2024-12-11 11:41:51,332 DEBUG SystemMonitor:743447 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-11 11:41:51,332 INFO HandlerThread:743447 [interfaces.py:finish():200] Joined disk monitor +2024-12-11 11:41:51,380 INFO HandlerThread:743447 [interfaces.py:finish():200] Joined gpu monitor +2024-12-11 11:41:51,381 INFO HandlerThread:743447 [interfaces.py:finish():200] Joined memory monitor +2024-12-11 11:41:51,381 INFO HandlerThread:743447 [interfaces.py:finish():200] Joined network monitor +2024-12-11 11:41:51,381 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:41:51,381 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-11 11:41:51,381 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 3 +2024-12-11 11:41:51,381 DEBUG SenderThread:743447 [sender.py:send():379] send: stats +2024-12-11 11:41:51,381 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:41:51,382 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-11 11:41:51,382 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:41:51,382 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-11 11:41:51,382 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 4 +2024-12-11 11:41:51,382 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:41:51,382 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-11 11:41:52,010 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:41:52,011 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json +2024-12-11 11:41:53,011 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:57,384 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:41:57,384 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:57,384 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:41:57,388 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: keepalive +2024-12-11 11:41:57,388 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-11 11:41:57,388 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 5 +2024-12-11 11:41:57,389 DEBUG SenderThread:743447 [sender.py:send():379] send: history +2024-12-11 11:41:57,389 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:41:57,389 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:41:57,389 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-11 11:41:57,389 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:41:57,389 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:41:57,390 DEBUG SenderThread:743447 [sender.py:send():379] send: history +2024-12-11 11:41:57,390 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:41:57,390 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:41:57,390 DEBUG SenderThread:743447 [sender.py:send():379] send: summary +2024-12-11 11:41:57,391 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:41:57,391 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:41:57,391 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-11 11:41:57,391 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 6 +2024-12-11 11:41:57,391 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:41:57,391 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-11 11:41:57,391 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:41:57,392 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-11 11:41:57,392 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 7 +2024-12-11 11:41:57,392 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:57,392 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:41:57,392 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-11 11:41:57,392 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:41:57,392 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-11 11:41:58,012 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json +2024-12-11 11:41:58,015 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 8 +2024-12-11 11:41:58,015 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:41:58,015 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-11 11:41:58,015 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:41:58,015 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-11 11:41:58,015 INFO SenderThread:743447 [job_builder.py:build():440] Attempting to build job artifact +2024-12-11 11:41:58,017 INFO SenderThread:743447 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-11 11:41:58,032 INFO SenderThread:743447 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-11 11:41:58,046 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 9 +2024-12-11 11:41:58,046 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:41:58,047 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-11 11:41:58,047 DEBUG SenderThread:743447 [sender.py:send():379] send: artifact +2024-12-11 11:41:58,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:41:58,562 INFO wandb-upload_1:743447 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpe19lto7x +2024-12-11 11:41:58,562 INFO wandb-upload_2:743447 [upload_job.py:push():85] Skipped uploading /tmp/tmpybw54_lu/wandb-job.json +2024-12-11 11:41:58,687 INFO wandb-upload_0:743447 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpugla6_vr +2024-12-11 11:41:59,013 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:59,244 INFO SenderThread:743447 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM2NzYwNTg3Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM2NzYwMDc2Nw==', 'versionIndex': 364}}} +2024-12-11 11:41:59,245 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:41:59,245 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-11 11:41:59,245 INFO SenderThread:743447 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-11 11:42:00,013 INFO SenderThread:743447 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files +2024-12-11 11:42:00,013 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/config.yaml config.yaml +2024-12-11 11:42:00,014 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:42:00,015 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json wandb-summary.json +2024-12-11 11:42:00,015 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/requirements.txt requirements.txt +2024-12-11 11:42:00,016 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-metadata.json wandb-metadata.json +2024-12-11 11:42:00,016 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log output.log +2024-12-11 11:42:00,016 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/diff.patch diff.patch +2024-12-11 11:42:00,016 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-11 11:42:00,022 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/0_eval_0_dea3cb7ce16ad057bd5f.mp4 media/videos/eval/0_eval_0_dea3cb7ce16ad057bd5f.mp4 +2024-12-11 11:42:00,022 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/4_eval_4_10441113b4d3ea44e8c6.mp4 media/videos/eval/4_eval_4_10441113b4d3ea44e8c6.mp4 +2024-12-11 11:42:00,022 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/2_eval_2_ddba0e3583786ac1097b.mp4 media/videos/eval/2_eval_2_ddba0e3583786ac1097b.mp4 +2024-12-11 11:42:00,022 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/3_eval_3_b687720b2d32aeeb9a91.mp4 media/videos/eval/3_eval_3_b687720b2d32aeeb9a91.mp4 +2024-12-11 11:42:00,022 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/1_eval_1_925210ff3b36d88f3887.mp4 media/videos/eval/1_eval_1_925210ff3b36d88f3887.mp4 +2024-12-11 11:42:00,022 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 10 +2024-12-11 11:42:00,023 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:42:00,023 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:42:00,023 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-11 11:42:00,023 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:42:00,023 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-11 11:42:00,023 INFO SenderThread:743447 [file_pusher.py:finish():169] shutting down file pusher +2024-12-11 11:42:00,215 INFO wandb-upload_1:743447 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/config.yaml +2024-12-11 11:42:00,233 INFO wandb-upload_2:743447 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json +2024-12-11 11:42:00,247 INFO wandb-upload_3:743447 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/requirements.txt +2024-12-11 11:42:00,264 INFO wandb-upload_4:743447 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:42:00,332 INFO wandb-upload_0:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/o5djqyqe-events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:42:00,332 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:42:00,333 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:42:00,532 INFO Thread-11 (_thread_body):743447 [sender.py:transition_state():614] send defer: 11 +2024-12-11 11:42:00,533 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:42:00,533 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-11 11:42:00,533 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:42:00,533 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-11 11:42:00,533 INFO SenderThread:743447 [file_pusher.py:join():175] waiting for file pusher +2024-12-11 11:42:00,534 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 12 +2024-12-11 11:42:00,534 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:42:00,534 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-11 11:42:00,534 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:42:00,534 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-11 11:42:00,534 INFO SenderThread:743447 [file_stream.py:finish():601] file stream finish called +2024-12-11 11:42:00,730 INFO SenderThread:743447 [file_stream.py:finish():605] file stream finish is done +2024-12-11 11:42:00,731 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 13 +2024-12-11 11:42:00,731 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:42:00,731 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-11 11:42:00,731 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:42:00,731 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-11 11:42:00,731 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 14 +2024-12-11 11:42:00,731 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:42:00,731 DEBUG SenderThread:743447 [sender.py:send():379] send: final +2024-12-11 11:42:00,731 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-11 11:42:00,731 DEBUG SenderThread:743447 [sender.py:send():379] send: footer +2024-12-11 11:42:00,731 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:42:00,731 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-11 11:42:00,732 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:42:00,732 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:42:00,732 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: server_info +2024-12-11 11:42:00,732 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: server_info +2024-12-11 11:42:00,734 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:42:00,734 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: get_summary +2024-12-11 11:42:00,734 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-11 11:42:00,735 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:42:00,795 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:42:00,795 INFO MainThread:743447 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-11 11:42:00,796 INFO MainThread:743447 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-11 11:42:00,796 INFO MainThread:743447 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-11 11:42:00,796 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: shutdown +2024-12-11 11:42:00,796 INFO HandlerThread:743447 [handler.py:finish():882] shutting down handler +2024-12-11 11:42:01,734 INFO WriterThread:743447 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/run-ahosb0lo.wandb +2024-12-11 11:42:01,795 INFO SenderThread:743447 [sender.py:finish():1615] shutting down sender +2024-12-11 11:42:01,795 INFO SenderThread:743447 [file_pusher.py:finish():169] shutting down file pusher +2024-12-11 11:42:01,795 INFO SenderThread:743447 [file_pusher.py:join():175] waiting for file pusher diff --git a/113638/wandb/debug.log b/113638/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d2fa4993882c1ade02b23b386433f0dd0e9e6a52 --- /dev/null +++ b/113638/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-11 11:36:44,614 INFO MainThread:743294 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-11 11:36:44,614 INFO MainThread:743294 [wandb_setup.py:_flush():76] Configure stats pid to 743294 +2024-12-11 11:36:44,614 INFO MainThread:743294 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-11 11:36:44,614 INFO MainThread:743294 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/settings +2024-12-11 11:36:44,614 INFO MainThread:743294 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-11 11:36:44,614 INFO MainThread:743294 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-11 11:36:44,614 INFO MainThread:743294 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-11 11:36:44,614 INFO MainThread:743294 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-11 11:36:44,614 INFO MainThread:743294 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/logs/debug.log +2024-12-11 11:36:44,614 INFO MainThread:743294 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/logs/debug-internal.log +2024-12-11 11:36:44,614 INFO MainThread:743294 [wandb_init.py:init():569] calling init triggers +2024-12-11 11:36:44,615 INFO MainThread:743294 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 2, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '265650', 'wandb_notes': '265650_req_265757restarted_2', 'eval': True, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim_left_to_right'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 16, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/265650', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'obs_shape': [14, 240, 320], 'action_shape': [36, 7], 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'true_action_history': False, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt'} +2024-12-11 11:36:44,615 INFO MainThread:743294 [wandb_init.py:init():619] starting backend +2024-12-11 11:36:44,615 INFO MainThread:743294 [wandb_init.py:init():623] setting up manager +2024-12-11 11:36:44,619 INFO MainThread:743294 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-11 11:36:44,620 INFO MainThread:743294 [wandb_init.py:init():631] backend started and connected +2024-12-11 11:36:44,632 INFO MainThread:743294 [wandb_init.py:init():720] updated telemetry +2024-12-11 11:36:44,639 INFO MainThread:743294 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-11 11:36:44,920 INFO MainThread:743294 [wandb_run.py:_on_init():2435] communicating current version +2024-12-11 11:36:44,991 INFO MainThread:743294 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.0 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-11 11:36:44,991 INFO MainThread:743294 [wandb_init.py:init():804] starting run threads in backend +2024-12-11 11:36:45,329 INFO MainThread:743294 [wandb_run.py:_console_start():2413] atexit reg +2024-12-11 11:36:45,329 INFO MainThread:743294 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-11 11:36:45,329 INFO MainThread:743294 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-11 11:36:45,329 INFO MainThread:743294 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-11 11:36:45,331 INFO MainThread:743294 [wandb_init.py:init():847] run started, returning control to user process +2024-12-11 11:36:45,331 INFO MainThread:743294 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/tb, True +2024-12-11 11:36:49,564 INFO MainThread:743294 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-11 11:42:01,796 WARNING MsgRouterThr:743294 [router.py:message_loop():77] message_loop has been closed diff --git a/113638/wandb/run-20241211_113644-ahosb0lo/files/code/FISH/eval_robot.py b/113638/wandb/run-20241211_113644-ahosb0lo/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..75a95b02db44bf1d6cdce4329109f52a0c4c1bc8 --- /dev/null +++ b/113638/wandb/run-20241211_113644-ahosb0lo/files/code/FISH/eval_robot.py @@ -0,0 +1,512 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/113638/wandb/run-20241211_113644-ahosb0lo/files/config.yaml b/113638/wandb/run-20241211_113644-ahosb0lo/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..042003ee68f99b67384cde0dac45c2b25e1dce42 --- /dev/null +++ b/113638/wandb/run-20241211_113644-ahosb0lo/files/config.yaml @@ -0,0 +1,966 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 2 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '265650' +wandb_notes: + desc: null + value: 265650_req_265757restarted_2 +eval: + desc: null + value: true +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: + - hbm_twodim_left_to_right +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 16 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/265650 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + obs_shape: + - 14 + - 240 + - 320 + action_shape: + - 36 + - 7 + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + policy_frequency: 5 + interpolation_frequency: 25 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + policy_frequency: 5 + interpolation_frequency: 25 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +true_action_history: + desc: null + value: false +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1733935004 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 16 + - 23 + - 35 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/113638/wandb/run-20241211_113644-ahosb0lo/files/diff.patch b/113638/wandb/run-20241211_113644-ahosb0lo/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..18af4fb4522440a64241cd32aaa593cf5121165f --- /dev/null +++ b/113638/wandb/run-20241211_113644-ahosb0lo/files/diff.patch @@ -0,0 +1,48 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 91f9f99..78e1f6e 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -124,18 +124,41 @@ bc_weight_type: 'qfilter' # linear, qfilter + # Load weights + load_checkpoint: ${agent.load_checkpoint} + # 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +-# full dataset ++# full 32 demos + # wandb_run_id: "265626" # RGBD+Mask+act history seed 0 + # wandb_run_id: "265638" # RGBD+Mask+act history seed 1 + # wandb_run_id: "265637" # RGBD+Mask+act history seed 2 + # wandb_run_id: "265636" # RGBD+Mask+act history seed 3 + # wandb_run_id: "265635" # RGBD+Mask+act history seed 4 + ++# 8 held out demos (26 demos) ++# wandb_run_id: "265652" # RGBD+Mask+contact+act history dataset seed 0 ++# wandb_run_id: "265654" # RGBD+Mask+contact+act history dataset seed 1 ++# wandb_run_id: "265655" # RGBD+Mask+contact+act history dataset seed 2 ++ ++# 8 held out demos (26 demos) ++# wandb_run_id: "265656" # RGBD+Mask+act history dataset seed 0 ++# wandb_run_id: "265657" # RGBD+Mask+act history dataset seed 1 ++# wandb_run_id: "265658" # RGBD+Mask+act history dataset seed 2 ++ ++# 16 held out demos (16 demos) ++# wandb_run_id: "265641" # RGBD+Mask+act history dataset seed 0 ++# wandb_run_id: "265644" # RGBD+Mask+act history dataset seed 1 ++# wandb_run_id: "265645" # RGBD+Mask+act history dataset seed 2 ++# wandb_run_id: "266381_0" # RGBD+Mask+act history dataset seed 4 ++ ++# 16 held out demos (16 demos) ++# wandb_run_id: "265646" # RGBD+Mask+Contact+act history dataset seed 0 ++# wandb_run_id: "265649" # RGBD+Mask+Contact+act history dataset seed 1 ++wandb_run_id: "265650" # RGBD+Mask+Contact+act history dataset seed 2 ++# wandb_run_id: "266378_0" # RGBD+Mask+Contact+act history dataset seed 4 ++ + # wandb_run_id: "265628" # RGBD+Mask+contact+act history seed 0 + # wandb_run_id: "265630" # RGBD+Mask+contact+act history seed 1 + # wandb_run_id: "265631" # RGBD+Mask+contact+act history seed 2 + # wandb_run_id: "265633" # RGBD+Mask+contact+act history seed 3 +-wandb_run_id: "265634" # RGBD+Mask+contact+act history seed 4 ++# wandb_run_id: "265634" # RGBD+Mask+contact+act history seed 4 ++ + + # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # full dataset diff --git a/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/0_eval_0_dea3cb7ce16ad057bd5f.mp4 b/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/0_eval_0_dea3cb7ce16ad057bd5f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6752b9c0bed742a62ee0ef6418806fd2c76c1b4e --- /dev/null +++ b/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/0_eval_0_dea3cb7ce16ad057bd5f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dea3cb7ce16ad057bd5fcb35b50592a86feeb9f7d6f3bd7b6f42095d27997931 +size 1422243 diff --git a/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/1_eval_1_925210ff3b36d88f3887.mp4 b/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/1_eval_1_925210ff3b36d88f3887.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..93c254d4f258ecfa7174a6a0bccd93ad07680a83 --- /dev/null +++ b/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/1_eval_1_925210ff3b36d88f3887.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:925210ff3b36d88f3887c6bf561b993eeb2172f9b898edf7efaa91f7c720e590 +size 1468875 diff --git a/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/2_eval_2_ddba0e3583786ac1097b.mp4 b/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/2_eval_2_ddba0e3583786ac1097b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a4b782d08953681dc476750407ad8b71b6e1ec08 --- /dev/null +++ b/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/2_eval_2_ddba0e3583786ac1097b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddba0e3583786ac1097bd8d918d5801eb4867cac0e908be1695101412c576aec +size 1498755 diff --git a/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/3_eval_3_b687720b2d32aeeb9a91.mp4 b/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/3_eval_3_b687720b2d32aeeb9a91.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a5831c1b674457255f4f48885d3fde460c528a80 --- /dev/null +++ b/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/3_eval_3_b687720b2d32aeeb9a91.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b687720b2d32aeeb9a91a18ae8924160e7860fcc22b2850d7f2b3b59ca27c2ec +size 1454836 diff --git a/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/4_eval_4_10441113b4d3ea44e8c6.mp4 b/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/4_eval_4_10441113b4d3ea44e8c6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7ce87e470e0e1a9f82045e8a1686732a7fe0cfa3 --- /dev/null +++ b/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/4_eval_4_10441113b4d3ea44e8c6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10441113b4d3ea44e8c6df553d629edb8882f588fa5814fc108ba5370ac0ec0d +size 1451627 diff --git a/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log b/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..f26b531f6f268aa463fb8e970a5d6dd5a76a5ec3 --- /dev/null +++ b/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log @@ -0,0 +1,6345 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1733935009.565167]: resetting environment +[INFO] [1733935009.573431]: cleared current plan +[INFO] [1733935009.575276]: moving to home +[INFO] [1733935010.577876]: reached home +[INFO] [1733935010.579581]: reset action history +[INFO] [1733935013.288663]: environment reset +Starting episode 0 +[INFO] [1733935013.292969]: resetting environment +[INFO] [1733935013.297452]: cleared current plan +[INFO] [1733935013.301012]: moving to home +[INFO] [1733935014.304699]: reached home +[INFO] [1733935014.306894]: reset action history +[INFO] [1733935017.016739]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1733935037.804429]: resetting environment +[INFO] [1733935037.808894]: cleared current plan +[INFO] [1733935037.810133]: moving to home +[INFO] [1733935038.811833]: reached home +[INFO] [1733935038.813154]: reset action history +[INFO] [1733935041.521652]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/episode_rosbags/episode_0_2024-12-11-11-37-21.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-743294.log +started roslaunch server http://158.130.50.37:36553/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [743589] +process[print_text-2]: started with pid [743613] +process[rosbag_record-3]: started with pid [743614] +started bagging! +[INFO] [1733935042.269685]: received mask_obs with header seq: 21450 +stamp: + secs: 1733935042 + nsecs: 273366212 +frame_id: '' and grasped_object_mask header seq: 501777 +stamp: + secs: 1733935042 + nsecs: 61149597 +frame_id: "camera_color_optical_frame" +[INFO] [1733935042.507709]: received mask_obs with header seq: 21451 +stamp: + secs: 1733935042 + nsecs: 511672735 +frame_id: '' and grasped_object_mask header seq: 501784 +stamp: + secs: 1733935042 + nsecs: 294291735 +frame_id: "camera_color_optical_frame" +[INFO] [1733935043.019073]: received mask_obs with header seq: 21452 +stamp: + secs: 1733935043 + nsecs: 22643089 +frame_id: '' and grasped_object_mask header seq: 501799 +stamp: + secs: 1733935042 + nsecs: 793879509 +frame_id: "camera_color_optical_frame" +[INFO] [1733935043.264069]: received mask_obs with header seq: 21453 +stamp: + secs: 1733935043 + nsecs: 267870187 +frame_id: '' and grasped_object_mask header seq: 501807 +stamp: + secs: 1733935043 + nsecs: 60324430 +frame_id: "camera_color_optical_frame" +[INFO] [1733935043.500366]: received mask_obs with header seq: 21454 +stamp: + secs: 1733935043 + nsecs: 504350900 +frame_id: '' and grasped_object_mask header seq: 501814 +stamp: + secs: 1733935043 + nsecs: 293463469 +frame_id: "camera_color_optical_frame" +[INFO] [1733935043.744718]: received mask_obs with header seq: 21455 +stamp: + secs: 1733935043 + nsecs: 748490571 +frame_id: '' and grasped_object_mask header seq: 501821 +stamp: + secs: 1733935043 + nsecs: 526602745 +frame_id: "camera_color_optical_frame" +[INFO] [1733935043.987580]: received mask_obs with header seq: 21456 +stamp: + secs: 1733935043 + nsecs: 991591215 +frame_id: '' and grasped_object_mask header seq: 501828 +stamp: + secs: 1733935043 + nsecs: 759745836 +frame_id: "camera_color_optical_frame" +[INFO] [1733935044.229017]: received mask_obs with header seq: 21457 +stamp: + secs: 1733935044 + nsecs: 233090400 +frame_id: '' and grasped_object_mask header seq: 501835 +stamp: + secs: 1733935043 + nsecs: 992889166 +frame_id: "camera_color_optical_frame" +[INFO] [1733935044.488958]: received mask_obs with header seq: 21458 +stamp: + secs: 1733935044 + nsecs: 492948770 +frame_id: '' and grasped_object_mask header seq: 501843 +stamp: + secs: 1733935044 + nsecs: 259337425 +frame_id: "camera_color_optical_frame" +[INFO] [1733935044.728161]: received mask_obs with header seq: 21459 +stamp: + secs: 1733935044 + nsecs: 732321023 +frame_id: '' and grasped_object_mask header seq: 501850 +stamp: + secs: 1733935044 + nsecs: 492479086 +frame_id: "camera_color_optical_frame" +[INFO] [1733935044.987720]: received mask_obs with header seq: 21460 +stamp: + secs: 1733935044 + nsecs: 991907596 +frame_id: '' and grasped_object_mask header seq: 501858 +stamp: + secs: 1733935044 + nsecs: 758926868 +frame_id: "camera_color_optical_frame" +[INFO] [1733935045.246821]: received mask_obs with header seq: 21461 +stamp: + secs: 1733935045 + nsecs: 250983953 +frame_id: '' and grasped_object_mask header seq: 501866 +stamp: + secs: 1733935045 + nsecs: 25373220 +frame_id: "camera_color_optical_frame" +[INFO] [1733935045.507641]: received mask_obs with header seq: 21462 +stamp: + secs: 1733935045 + nsecs: 511660337 +frame_id: '' and grasped_object_mask header seq: 501874 +stamp: + secs: 1733935045 + nsecs: 291819334 +frame_id: "camera_color_optical_frame" +[INFO] [1733935045.764531]: received mask_obs with header seq: 21463 +stamp: + secs: 1733935045 + nsecs: 768618822 +frame_id: '' and grasped_object_mask header seq: 501881 +stamp: + secs: 1733935045 + nsecs: 524960041 +frame_id: "camera_color_optical_frame" +[INFO] [1733935046.020351]: received mask_obs with header seq: 21464 +stamp: + secs: 1733935046 + nsecs: 24522066 +frame_id: '' and grasped_object_mask header seq: 501889 +stamp: + secs: 1733935045 + nsecs: 791410685 +frame_id: "camera_color_optical_frame" +[INFO] [1733935046.278242]: received mask_obs with header seq: 21465 +stamp: + secs: 1733935046 + nsecs: 282319307 +frame_id: '' and grasped_object_mask header seq: 501897 +stamp: + secs: 1733935046 + nsecs: 57861328 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -101762986 for nearest: 1733935046222548986 - target: 1733935046324311972 at idx 73 +[INFO] [1733935046.542843]: received mask_obs with header seq: 21466 +stamp: + secs: 1733935046 + nsecs: 546957492 +frame_id: '' and grasped_object_mask header seq: 501905 +stamp: + secs: 1733935046 + nsecs: 324311972 +frame_id: "camera_color_optical_frame" +[INFO] [1733935046.799392]: received mask_obs with header seq: 21467 +stamp: + secs: 1733935046 + nsecs: 803374052 +frame_id: '' and grasped_object_mask header seq: 501912 +stamp: + secs: 1733935046 + nsecs: 557456970 +frame_id: "camera_color_optical_frame" +[INFO] [1733935047.042510]: received mask_obs with header seq: 21468 +stamp: + secs: 1733935047 + nsecs: 46567440 +frame_id: '' and grasped_object_mask header seq: 501920 +stamp: + secs: 1733935046 + nsecs: 823908806 +frame_id: "camera_color_optical_frame" +[INFO] [1733935047.282103]: received mask_obs with header seq: 21469 +stamp: + secs: 1733935047 + nsecs: 285995483 +frame_id: '' and grasped_object_mask header seq: 501927 +stamp: + secs: 1733935047 + nsecs: 57054281 +frame_id: "camera_color_optical_frame" +[INFO] [1733935047.517807]: received mask_obs with header seq: 21470 +stamp: + secs: 1733935047 + nsecs: 521833896 +frame_id: '' and grasped_object_mask header seq: 501934 +stamp: + secs: 1733935047 + nsecs: 290199995 +frame_id: "camera_color_optical_frame" +[INFO] [1733935047.762105]: received mask_obs with header seq: 21471 +stamp: + secs: 1733935047 + nsecs: 766016483 +frame_id: '' and grasped_object_mask header seq: 501941 +stamp: + secs: 1733935047 + nsecs: 523344755 +frame_id: "camera_color_optical_frame" +[INFO] [1733935048.018344]: received mask_obs with header seq: 21472 +stamp: + secs: 1733935048 + nsecs: 21991491 +frame_id: '' and grasped_object_mask header seq: 501949 +stamp: + secs: 1733935047 + nsecs: 789792776 +frame_id: "camera_color_optical_frame" +[INFO] [1733935048.279607]: received mask_obs with header seq: 21473 +stamp: + secs: 1733935048 + nsecs: 283287048 +frame_id: '' and grasped_object_mask header seq: 501957 +stamp: + secs: 1733935048 + nsecs: 56239128 +frame_id: "camera_color_optical_frame" +[INFO] [1733935048.545446]: received mask_obs with header seq: 21474 +stamp: + secs: 1733935048 + nsecs: 549492597 +frame_id: '' and grasped_object_mask header seq: 501965 +stamp: + secs: 1733935048 + nsecs: 322685003 +frame_id: "camera_color_optical_frame" +[INFO] [1733935048.791651]: received mask_obs with header seq: 21475 +stamp: + secs: 1733935048 + nsecs: 795744180 +frame_id: '' and grasped_object_mask header seq: 501972 +stamp: + secs: 1733935048 + nsecs: 555825710 +frame_id: "camera_color_optical_frame" +[INFO] [1733935049.047181]: received mask_obs with header seq: 21476 +stamp: + secs: 1733935049 + nsecs: 50847530 +frame_id: '' and grasped_object_mask header seq: 501980 +stamp: + secs: 1733935048 + nsecs: 822274446 +frame_id: "camera_color_optical_frame" +[INFO] [1733935049.305209]: received mask_obs with header seq: 21477 +stamp: + secs: 1733935049 + nsecs: 308900356 +frame_id: '' and grasped_object_mask header seq: 501988 +stamp: + secs: 1733935049 + nsecs: 88723183 +frame_id: "camera_color_optical_frame" +[INFO] [1733935049.544435]: received mask_obs with header seq: 21478 +stamp: + secs: 1733935049 + nsecs: 548217296 +frame_id: '' and grasped_object_mask header seq: 501995 +stamp: + secs: 1733935049 + nsecs: 321866035 +frame_id: "camera_color_optical_frame" +[INFO] [1733935049.786714]: received mask_obs with header seq: 21479 +stamp: + secs: 1733935049 + nsecs: 790311336 +frame_id: '' and grasped_object_mask header seq: 502002 +stamp: + secs: 1733935049 + nsecs: 555009365 +frame_id: "camera_color_optical_frame" +[INFO] [1733935050.033924]: received mask_obs with header seq: 21480 +stamp: + secs: 1733935050 + nsecs: 37808418 +frame_id: '' and grasped_object_mask header seq: 502009 +stamp: + secs: 1733935049 + nsecs: 788157225 +frame_id: "camera_color_optical_frame" +[INFO] [1733935050.303283]: received mask_obs with header seq: 21481 +stamp: + secs: 1733935050 + nsecs: 306779623 +frame_id: '' and grasped_object_mask header seq: 502017 +stamp: + secs: 1733935050 + nsecs: 54613113 +frame_id: "camera_color_optical_frame" +[INFO] [1733935050.564012]: received mask_obs with header seq: 21482 +stamp: + secs: 1733935050 + nsecs: 568029642 +frame_id: '' and grasped_object_mask header seq: 502025 +stamp: + secs: 1733935050 + nsecs: 321070194 +frame_id: "camera_color_optical_frame" +[INFO] [1733935050.816839]: received mask_obs with header seq: 21483 +stamp: + secs: 1733935050 + nsecs: 820862770 +frame_id: '' and grasped_object_mask header seq: 502033 +stamp: + secs: 1733935050 + nsecs: 587527752 +frame_id: "camera_color_optical_frame" +[INFO] [1733935051.070821]: received mask_obs with header seq: 21484 +stamp: + secs: 1733935051 + nsecs: 74638366 +frame_id: '' and grasped_object_mask header seq: 502040 +stamp: + secs: 1733935050 + nsecs: 820681334 +frame_id: "camera_color_optical_frame" +[INFO] [1733935051.318069]: received mask_obs with header seq: 21485 +stamp: + secs: 1733935051 + nsecs: 322130680 +frame_id: '' and grasped_object_mask header seq: 502048 +stamp: + secs: 1733935051 + nsecs: 87142706 +frame_id: "camera_color_optical_frame" +[INFO] [1733935051.558905]: received mask_obs with header seq: 21486 +stamp: + secs: 1733935051 + nsecs: 562958955 +frame_id: '' and grasped_object_mask header seq: 502055 +stamp: + secs: 1733935051 + nsecs: 320296764 +frame_id: "camera_color_optical_frame" +[INFO] [1733935051.802352]: received mask_obs with header seq: 21487 +stamp: + secs: 1733935051 + nsecs: 804764747 +frame_id: '' and grasped_object_mask header seq: 502063 +stamp: + secs: 1733935051 + nsecs: 586757421 +frame_id: "camera_color_optical_frame" +[INFO] [1733935052.058118]: received mask_obs with header seq: 21488 +stamp: + secs: 1733935052 + nsecs: 62005281 +frame_id: '' and grasped_object_mask header seq: 502070 +stamp: + secs: 1733935051 + nsecs: 819891691 +frame_id: "camera_color_optical_frame" +[INFO] [1733935052.306707]: received mask_obs with header seq: 21489 +stamp: + secs: 1733935052 + nsecs: 310359239 +frame_id: '' and grasped_object_mask header seq: 502078 +stamp: + secs: 1733935052 + nsecs: 86331844 +frame_id: "camera_color_optical_frame" +[INFO] [1733935052.550337]: received mask_obs with header seq: 21490 +stamp: + secs: 1733935052 + nsecs: 554329633 +frame_id: '' and grasped_object_mask header seq: 502085 +stamp: + secs: 1733935052 + nsecs: 319465160 +frame_id: "camera_color_optical_frame" +[INFO] [1733935052.786188]: received mask_obs with header seq: 21491 +stamp: + secs: 1733935052 + nsecs: 789936065 +frame_id: '' and grasped_object_mask header seq: 502092 +stamp: + secs: 1733935052 + nsecs: 552599192 +frame_id: "camera_color_optical_frame" +[INFO] [1733935053.033528]: received mask_obs with header seq: 21492 +stamp: + secs: 1733935053 + nsecs: 37027597 +frame_id: '' and grasped_object_mask header seq: 502100 +stamp: + secs: 1733935052 + nsecs: 819049358 +frame_id: "camera_color_optical_frame" +[INFO] [1733935053.286218]: received mask_obs with header seq: 21493 +stamp: + secs: 1733935053 + nsecs: 290180206 +frame_id: '' and grasped_object_mask header seq: 502107 +stamp: + secs: 1733935053 + nsecs: 52194118 +frame_id: "camera_color_optical_frame" +[INFO] [1733935053.541406]: received mask_obs with header seq: 21494 +stamp: + secs: 1733935053 + nsecs: 545487880 +frame_id: '' and grasped_object_mask header seq: 502115 +stamp: + secs: 1733935053 + nsecs: 318644524 +frame_id: "camera_color_optical_frame" +[INFO] [1733935053.782285]: received mask_obs with header seq: 21495 +stamp: + secs: 1733935053 + nsecs: 786401748 +frame_id: '' and grasped_object_mask header seq: 502122 +stamp: + secs: 1733935053 + nsecs: 551789284 +frame_id: "camera_color_optical_frame" +[INFO] [1733935054.020615]: received mask_obs with header seq: 21496 +stamp: + secs: 1733935054 + nsecs: 24511337 +frame_id: '' and grasped_object_mask header seq: 502129 +stamp: + secs: 1733935053 + nsecs: 784925938 +frame_id: "camera_color_optical_frame" +[INFO] [1733935054.277713]: received mask_obs with header seq: 21497 +stamp: + secs: 1733935054 + nsecs: 281760692 +frame_id: '' and grasped_object_mask header seq: 502137 +stamp: + secs: 1733935054 + nsecs: 51368952 +frame_id: "camera_color_optical_frame" +[INFO] [1733935054.513986]: received mask_obs with header seq: 21498 +stamp: + secs: 1733935054 + nsecs: 518069505 +frame_id: '' and grasped_object_mask header seq: 502144 +stamp: + secs: 1733935054 + nsecs: 284504414 +frame_id: "camera_color_optical_frame" +[INFO] [1733935054.766434]: received mask_obs with header seq: 21499 +stamp: + secs: 1733935054 + nsecs: 770146608 +frame_id: '' and grasped_object_mask header seq: 502151 +stamp: + secs: 1733935054 + nsecs: 517640591 +frame_id: "camera_color_optical_frame" +[INFO] [1733935055.023534]: received mask_obs with header seq: 21500 +stamp: + secs: 1733935055 + nsecs: 27426958 +frame_id: '' and grasped_object_mask header seq: 502159 +stamp: + secs: 1733935054 + nsecs: 784095764 +frame_id: "camera_color_optical_frame" +[INFO] [1733935055.281309]: received mask_obs with header seq: 21501 +stamp: + secs: 1733935055 + nsecs: 285436153 +frame_id: '' and grasped_object_mask header seq: 502167 +stamp: + secs: 1733935055 + nsecs: 50554276 +frame_id: "camera_color_optical_frame" +[INFO] [1733935055.557713]: received mask_obs with header seq: 21502 +stamp: + secs: 1733935055 + nsecs: 561653137 +frame_id: '' and grasped_object_mask header seq: 502175 +stamp: + secs: 1733935055 + nsecs: 317012310 +frame_id: "camera_color_optical_frame" +[INFO] [1733935055.796848]: received mask_obs with header seq: 21503 +stamp: + secs: 1733935055 + nsecs: 800902366 +frame_id: '' and grasped_object_mask header seq: 502183 +stamp: + secs: 1733935055 + nsecs: 583468914 +frame_id: "camera_color_optical_frame" +[INFO] [1733935056.055182]: received mask_obs with header seq: 21504 +stamp: + secs: 1733935056 + nsecs: 58822870 +frame_id: '' and grasped_object_mask header seq: 502190 +stamp: + secs: 1733935055 + nsecs: 816611528 +frame_id: "camera_color_optical_frame" +[INFO] [1733935056.311769]: received mask_obs with header seq: 21505 +stamp: + secs: 1733935056 + nsecs: 315775632 +frame_id: '' and grasped_object_mask header seq: 502198 +stamp: + secs: 1733935056 + nsecs: 83059788 +frame_id: "camera_color_optical_frame" +[INFO] [1733935056.549873]: received mask_obs with header seq: 21506 +stamp: + secs: 1733935056 + nsecs: 553673267 +frame_id: '' and grasped_object_mask header seq: 502205 +stamp: + secs: 1733935056 + nsecs: 316202402 +frame_id: "camera_color_optical_frame" +[INFO] [1733935056.795867]: received mask_obs with header seq: 21507 +stamp: + secs: 1733935056 + nsecs: 799722909 +frame_id: '' and grasped_object_mask header seq: 502212 +stamp: + secs: 1733935056 + nsecs: 549344063 +frame_id: "camera_color_optical_frame" +[INFO] [1733935057.047035]: received mask_obs with header seq: 21508 +stamp: + secs: 1733935057 + nsecs: 50943374 +frame_id: '' and grasped_object_mask header seq: 502220 +stamp: + secs: 1733935056 + nsecs: 815786362 +frame_id: "camera_color_optical_frame" +[INFO] [1733935057.292310]: received mask_obs with header seq: 21509 +stamp: + secs: 1733935057 + nsecs: 296411275 +frame_id: '' and grasped_object_mask header seq: 502227 +stamp: + secs: 1733935057 + nsecs: 48923492 +frame_id: "camera_color_optical_frame" +[INFO] [1733935057.556490]: received mask_obs with header seq: 21510 +stamp: + secs: 1733935057 + nsecs: 560437440 +frame_id: '' and grasped_object_mask header seq: 502235 +stamp: + secs: 1733935057 + nsecs: 315365076 +frame_id: "camera_color_optical_frame" +[INFO] [1733935057.815871]: received mask_obs with header seq: 21511 +stamp: + secs: 1733935057 + nsecs: 816685914 +frame_id: '' and grasped_object_mask header seq: 502243 +stamp: + secs: 1733935057 + nsecs: 581808090 +frame_id: "camera_color_optical_frame" +[INFO] [1733935058.082218]: received mask_obs with header seq: 21512 +stamp: + secs: 1733935058 + nsecs: 86357593 +frame_id: '' and grasped_object_mask header seq: 502251 +stamp: + secs: 1733935057 + nsecs: 848255873 +frame_id: "camera_color_optical_frame" +[INFO] [1733935058.344667]: received mask_obs with header seq: 21513 +stamp: + secs: 1733935058 + nsecs: 348472595 +frame_id: '' and grasped_object_mask header seq: 502259 +stamp: + secs: 1733935058 + nsecs: 114703655 +frame_id: "camera_color_optical_frame" +[INFO] [1733935058.597115]: received mask_obs with header seq: 21514 +stamp: + secs: 1733935058 + nsecs: 601005792 +frame_id: '' and grasped_object_mask header seq: 502267 +stamp: + secs: 1733935058 + nsecs: 381151199 +frame_id: "camera_color_optical_frame" +[INFO] [1733935058.840459]: received mask_obs with header seq: 21515 +stamp: + secs: 1733935058 + nsecs: 844274997 +frame_id: '' and grasped_object_mask header seq: 502274 +stamp: + secs: 1733935058 + nsecs: 614295244 +frame_id: "camera_color_optical_frame" +[INFO] [1733935059.106814]: received mask_obs with header seq: 21516 +stamp: + secs: 1733935059 + nsecs: 110725164 +frame_id: '' and grasped_object_mask header seq: 502282 +stamp: + secs: 1733935058 + nsecs: 880750418 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 108647545 for nearest: 1733935059222547014 - target: 1733935059113899469 at idx 243 +[INFO] [1733935059.363535]: received mask_obs with header seq: 21517 +stamp: + secs: 1733935059 + nsecs: 366582632 +frame_id: '' and grasped_object_mask header seq: 502289 +stamp: + secs: 1733935059 + nsecs: 113899469 +frame_id: "camera_color_optical_frame" +[INFO] [1733935059.615239]: received mask_obs with header seq: 21518 +stamp: + secs: 1733935059 + nsecs: 619328022 +frame_id: '' and grasped_object_mask header seq: 502297 +stamp: + secs: 1733935059 + nsecs: 380354643 +frame_id: "camera_color_optical_frame" +[INFO] [1733935059.874483]: received mask_obs with header seq: 21519 +stamp: + secs: 1733935059 + nsecs: 877950429 +frame_id: '' and grasped_object_mask header seq: 502305 +stamp: + secs: 1733935059 + nsecs: 646811247 +frame_id: "camera_color_optical_frame" +[INFO] [1733935060.135543]: received mask_obs with header seq: 21520 +stamp: + secs: 1733935060 + nsecs: 138919830 +frame_id: '' and grasped_object_mask header seq: 502313 +stamp: + secs: 1733935059 + nsecs: 913269281 +frame_id: "camera_color_optical_frame" +[INFO] [1733935060.393926]: received mask_obs with header seq: 21521 +stamp: + secs: 1733935060 + nsecs: 397425889 +frame_id: '' and grasped_object_mask header seq: 502321 +stamp: + secs: 1733935060 + nsecs: 179728746 +frame_id: "camera_color_optical_frame" +[INFO] [1733935060.644738]: received mask_obs with header seq: 21522 +stamp: + secs: 1733935060 + nsecs: 648850202 +frame_id: '' and grasped_object_mask header seq: 502328 +stamp: + secs: 1733935060 + nsecs: 412879705 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -156794067 for nearest: 1733935060522542958 - target: 1733935060679337025 at idx 249 +[INFO] [1733935060.904055]: received mask_obs with header seq: 21523 +stamp: + secs: 1733935060 + nsecs: 907726764 +frame_id: '' and grasped_object_mask header seq: 502336 +stamp: + secs: 1733935060 + nsecs: 679337025 +frame_id: "camera_color_optical_frame" +[INFO] [1733935061.142322]: received mask_obs with header seq: 21524 +stamp: + secs: 1733935061 + nsecs: 146464586 +frame_id: '' and grasped_object_mask header seq: 502343 +stamp: + secs: 1733935060 + nsecs: 912482977 +frame_id: "camera_color_optical_frame" +[INFO] [1733935061.395799]: received mask_obs with header seq: 21525 +stamp: + secs: 1733935061 + nsecs: 399906396 +frame_id: '' and grasped_object_mask header seq: 502351 +stamp: + secs: 1733935061 + nsecs: 178936243 +frame_id: "camera_color_optical_frame" +[INFO] [1733935061.637577]: received mask_obs with header seq: 21526 +stamp: + secs: 1733935061 + nsecs: 641685485 +frame_id: '' and grasped_object_mask header seq: 502358 +stamp: + secs: 1733935061 + nsecs: 412083387 +frame_id: "camera_color_optical_frame" +[INFO] [1733935061.907149]: received mask_obs with header seq: 21527 +stamp: + secs: 1733935061 + nsecs: 911283493 +frame_id: '' and grasped_object_mask header seq: 502366 +stamp: + secs: 1733935061 + nsecs: 678537846 +frame_id: "camera_color_optical_frame" +[INFO] [1733935062.165823]: received mask_obs with header seq: 21528 +stamp: + secs: 1733935062 + nsecs: 169265508 +frame_id: '' and grasped_object_mask header seq: 502374 +stamp: + secs: 1733935061 + nsecs: 944994450 +frame_id: "camera_color_optical_frame" +[INFO] [1733935062.426893]: received mask_obs with header seq: 21529 +stamp: + secs: 1733935062 + nsecs: 430837869 +frame_id: '' and grasped_object_mask header seq: 502382 +stamp: + secs: 1733935062 + nsecs: 211449862 +frame_id: "camera_color_optical_frame" +[INFO] [1733935062.684947]: received mask_obs with header seq: 21530 +stamp: + secs: 1733935062 + nsecs: 689020156 +frame_id: '' and grasped_object_mask header seq: 502389 +stamp: + secs: 1733935062 + nsecs: 444599628 +frame_id: "camera_color_optical_frame" +[INFO] [1733935062.941595]: received mask_obs with header seq: 21531 +stamp: + secs: 1733935062 + nsecs: 945550441 +frame_id: '' and grasped_object_mask header seq: 502397 +stamp: + secs: 1733935062 + nsecs: 711052418 +frame_id: "camera_color_optical_frame" +[INFO] [1733935063.198937]: received mask_obs with header seq: 21532 +stamp: + secs: 1733935063 + nsecs: 202995300 +frame_id: '' and grasped_object_mask header seq: 502405 +stamp: + secs: 1733935062 + nsecs: 977503777 +frame_id: "camera_color_optical_frame" +[INFO] [1733935063.435653]: received mask_obs with header seq: 21533 +stamp: + secs: 1733935063 + nsecs: 439588785 +frame_id: '' and grasped_object_mask header seq: 502412 +stamp: + secs: 1733935063 + nsecs: 210649252 +frame_id: "camera_color_optical_frame" +[INFO] [1733935063.672600]: received mask_obs with header seq: 21534 +stamp: + secs: 1733935063 + nsecs: 676503181 +frame_id: '' and grasped_object_mask header seq: 502419 +stamp: + secs: 1733935063 + nsecs: 443793535 +frame_id: "camera_color_optical_frame" +[INFO] [1733935063.910960]: received mask_obs with header seq: 21535 +stamp: + secs: 1733935063 + nsecs: 914987087 +frame_id: '' and grasped_object_mask header seq: 502426 +stamp: + secs: 1733935063 + nsecs: 676937342 +frame_id: "camera_color_optical_frame" +[INFO] [1733935064.159721]: received mask_obs with header seq: 21536 +stamp: + secs: 1733935064 + nsecs: 162232637 +frame_id: '' and grasped_object_mask header seq: 502434 +stamp: + secs: 1733935063 + nsecs: 943385363 +frame_id: "camera_color_optical_frame" +[INFO] [1733935064.411388]: received mask_obs with header seq: 21537 +stamp: + secs: 1733935064 + nsecs: 415457010 +frame_id: '' and grasped_object_mask header seq: 502441 +stamp: + secs: 1733935064 + nsecs: 176527500 +frame_id: "camera_color_optical_frame" +[INFO] [1733935064.659864]: received mask_obs with header seq: 21538 +stamp: + secs: 1733935064 + nsecs: 660864830 +frame_id: '' and grasped_object_mask header seq: 502449 +stamp: + secs: 1733935064 + nsecs: 442975044 +frame_id: "camera_color_optical_frame" +[INFO] [1733935064.916227]: received mask_obs with header seq: 21539 +stamp: + secs: 1733935064 + nsecs: 920123100 +frame_id: '' and grasped_object_mask header seq: 502456 +stamp: + secs: 1733935064 + nsecs: 676117420 +frame_id: "camera_color_optical_frame" +[INFO] [1733935065.167214]: received mask_obs with header seq: 21540 +stamp: + secs: 1733935065 + nsecs: 170959711 +frame_id: '' and grasped_object_mask header seq: 502464 +stamp: + secs: 1733935064 + nsecs: 942565203 +frame_id: "camera_color_optical_frame" +[INFO] [1733935065.404157]: received mask_obs with header seq: 21541 +stamp: + secs: 1733935065 + nsecs: 408244848 +frame_id: '' and grasped_object_mask header seq: 502471 +stamp: + secs: 1733935065 + nsecs: 175707579 +frame_id: "camera_color_optical_frame" +[INFO] [1733935065.654117]: received mask_obs with header seq: 21542 +stamp: + secs: 1733935065 + nsecs: 658243417 +frame_id: '' and grasped_object_mask header seq: 502479 +stamp: + secs: 1733935065 + nsecs: 442155361 +frame_id: "camera_color_optical_frame" +[INFO] [1733935065.908292]: received mask_obs with header seq: 21543 +stamp: + secs: 1733935065 + nsecs: 912242174 +frame_id: '' and grasped_object_mask header seq: 502486 +stamp: + secs: 1733935065 + nsecs: 675300598 +frame_id: "camera_color_optical_frame" +[INFO] [1733935066.149390]: received mask_obs with header seq: 21544 +stamp: + secs: 1733935066 + nsecs: 153505802 +frame_id: '' and grasped_object_mask header seq: 502493 +stamp: + secs: 1733935065 + nsecs: 908447027 +frame_id: "camera_color_optical_frame" +[INFO] [1733935066.404264]: received mask_obs with header seq: 21545 +stamp: + secs: 1733935066 + nsecs: 408416748 +frame_id: '' and grasped_object_mask header seq: 502501 +stamp: + secs: 1733935066 + nsecs: 174901962 +frame_id: "camera_color_optical_frame" +[INFO] [1733935066.642348]: received mask_obs with header seq: 21546 +stamp: + secs: 1733935066 + nsecs: 646448135 +frame_id: '' and grasped_object_mask header seq: 502508 +stamp: + secs: 1733935066 + nsecs: 408048868 +frame_id: "camera_color_optical_frame" +[INFO] [1733935066.899178]: received mask_obs with header seq: 21547 +stamp: + secs: 1733935066 + nsecs: 903156280 +frame_id: '' and grasped_object_mask header seq: 502516 +stamp: + secs: 1733935066 + nsecs: 674506187 +frame_id: "camera_color_optical_frame" +[INFO] [1733935067.159172]: received mask_obs with header seq: 21548 +stamp: + secs: 1733935067 + nsecs: 163305521 +frame_id: '' and grasped_object_mask header seq: 502524 +stamp: + secs: 1733935066 + nsecs: 940968037 +frame_id: "camera_color_optical_frame" +[INFO] [1733935067.421804]: received mask_obs with header seq: 21549 +stamp: + secs: 1733935067 + nsecs: 425332546 +frame_id: '' and grasped_object_mask header seq: 502532 +stamp: + secs: 1733935067 + nsecs: 207429647 +frame_id: "camera_color_optical_frame" +[INFO] [1733935067.681789]: received mask_obs with header seq: 21550 +stamp: + secs: 1733935067 + nsecs: 685898780 +frame_id: '' and grasped_object_mask header seq: 502539 +stamp: + secs: 1733935067 + nsecs: 440584660 +frame_id: "camera_color_optical_frame" +[INFO] [1733935067.929197]: received mask_obs with header seq: 21551 +stamp: + secs: 1733935067 + nsecs: 932657003 +frame_id: '' and grasped_object_mask header seq: 502547 +stamp: + secs: 1733935067 + nsecs: 707036734 +frame_id: "camera_color_optical_frame" +[INFO] [1733935068.167980]: received mask_obs with header seq: 21552 +stamp: + secs: 1733935068 + nsecs: 171568632 +frame_id: '' and grasped_object_mask header seq: 502554 +stamp: + secs: 1733935067 + nsecs: 940174103 +frame_id: "camera_color_optical_frame" +[INFO] [1733935068.406546]: received mask_obs with header seq: 21553 +stamp: + secs: 1733935068 + nsecs: 410364151 +frame_id: '' and grasped_object_mask header seq: 502561 +stamp: + secs: 1733935068 + nsecs: 173311949 +frame_id: "camera_color_optical_frame" +[INFO] [1733935068.668174]: received mask_obs with header seq: 21554 +stamp: + secs: 1733935068 + nsecs: 672015905 +frame_id: '' and grasped_object_mask header seq: 502569 +stamp: + secs: 1733935068 + nsecs: 439754248 +frame_id: "camera_color_optical_frame" +[INFO] [1733935068.925759]: received mask_obs with header seq: 21555 +stamp: + secs: 1733935068 + nsecs: 929679632 +frame_id: '' and grasped_object_mask header seq: 502577 +stamp: + secs: 1733935068 + nsecs: 706202984 +frame_id: "camera_color_optical_frame" +[INFO] [1733935069.164898]: received mask_obs with header seq: 21556 +stamp: + secs: 1733935069 + nsecs: 168161869 +frame_id: '' and grasped_object_mask header seq: 502584 +stamp: + secs: 1733935068 + nsecs: 939349651 +frame_id: "camera_color_optical_frame" +[INFO] [1733935069.403122]: received mask_obs with header seq: 21557 +stamp: + secs: 1733935069 + nsecs: 407195806 +frame_id: '' and grasped_object_mask header seq: 502591 +stamp: + secs: 1733935069 + nsecs: 172497034 +frame_id: "camera_color_optical_frame" +[INFO] [1733935069.639706]: received mask_obs with header seq: 21558 +stamp: + secs: 1733935069 + nsecs: 643782854 +frame_id: '' and grasped_object_mask header seq: 502598 +stamp: + secs: 1733935069 + nsecs: 405644894 +frame_id: "camera_color_optical_frame" +[INFO] [1733935069.884654]: received mask_obs with header seq: 21559 +stamp: + secs: 1733935069 + nsecs: 888788461 +frame_id: '' and grasped_object_mask header seq: 502606 +stamp: + secs: 1733935069 + nsecs: 672097683 +frame_id: "camera_color_optical_frame" +[INFO] [1733935070.138199]: received mask_obs with header seq: 21560 +stamp: + secs: 1733935070 + nsecs: 141910791 +frame_id: '' and grasped_object_mask header seq: 502613 +stamp: + secs: 1733935069 + nsecs: 905242205 +frame_id: "camera_color_optical_frame" +[INFO] [1733935070.378649]: received mask_obs with header seq: 21561 +stamp: + secs: 1733935070 + nsecs: 382622480 +frame_id: '' and grasped_object_mask header seq: 502620 +stamp: + secs: 1733935070 + nsecs: 138385534 +frame_id: "camera_color_optical_frame" +[INFO] [1733935070.634808]: received mask_obs with header seq: 21562 +stamp: + secs: 1733935070 + nsecs: 638879060 +frame_id: '' and grasped_object_mask header seq: 502628 +stamp: + secs: 1733935070 + nsecs: 404835701 +frame_id: "camera_color_optical_frame" +[INFO] [1733935070.889321]: received mask_obs with header seq: 21563 +stamp: + secs: 1733935070 + nsecs: 893135547 +frame_id: '' and grasped_object_mask header seq: 502636 +stamp: + secs: 1733935070 + nsecs: 671288967 +frame_id: "camera_color_optical_frame" +[INFO] [1733935071.127904]: received mask_obs with header seq: 21564 +stamp: + secs: 1733935071 + nsecs: 131193399 +frame_id: '' and grasped_object_mask header seq: 502643 +stamp: + secs: 1733935070 + nsecs: 904438019 +frame_id: "camera_color_optical_frame" +[INFO] [1733935071.367356]: received mask_obs with header seq: 21565 +stamp: + secs: 1733935071 + nsecs: 371166944 +frame_id: '' and grasped_object_mask header seq: 502650 +stamp: + secs: 1733935071 + nsecs: 137588024 +frame_id: "camera_color_optical_frame" +[INFO] [1733935071.622988]: received mask_obs with header seq: 21566 +stamp: + secs: 1733935071 + nsecs: 627081155 +frame_id: '' and grasped_object_mask header seq: 502658 +stamp: + secs: 1733935071 + nsecs: 404045343 +frame_id: "camera_color_optical_frame" +[INFO] [1733935071.861801]: received mask_obs with header seq: 21567 +stamp: + secs: 1733935071 + nsecs: 865952491 +frame_id: '' and grasped_object_mask header seq: 502665 +stamp: + secs: 1733935071 + nsecs: 637196302 +frame_id: "camera_color_optical_frame" +[INFO] [1733935072.100408]: received mask_obs with header seq: 21568 +stamp: + secs: 1733935072 + nsecs: 104532003 +frame_id: '' and grasped_object_mask header seq: 502672 +stamp: + secs: 1733935071 + nsecs: 870346308 +frame_id: "camera_color_optical_frame" +[INFO] [1733935072.346397]: received mask_obs with header seq: 21569 +stamp: + secs: 1733935072 + nsecs: 350338697 +frame_id: '' and grasped_object_mask header seq: 502679 +stamp: + secs: 1733935072 + nsecs: 103495121 +frame_id: "camera_color_optical_frame" +[INFO] [1733935072.602857]: received mask_obs with header seq: 21570 +stamp: + secs: 1733935072 + nsecs: 606857299 +frame_id: '' and grasped_object_mask header seq: 502687 +stamp: + secs: 1733935072 + nsecs: 369953156 +frame_id: "camera_color_optical_frame" +[INFO] [1733935072.862751]: received mask_obs with header seq: 21571 +stamp: + secs: 1733935072 + nsecs: 866791963 +frame_id: '' and grasped_object_mask header seq: 502695 +stamp: + secs: 1733935072 + nsecs: 636405945 +frame_id: "camera_color_optical_frame" +[INFO] [1733935073.119259]: received mask_obs with header seq: 21572 +stamp: + secs: 1733935073 + nsecs: 123228788 +frame_id: '' and grasped_object_mask header seq: 502703 +stamp: + secs: 1733935072 + nsecs: 902846098 +frame_id: "camera_color_optical_frame" +[INFO] [1733935073.382107]: received mask_obs with header seq: 21573 +stamp: + secs: 1733935073 + nsecs: 385686159 +frame_id: '' and grasped_object_mask header seq: 502711 +stamp: + secs: 1733935073 + nsecs: 169285536 +frame_id: "camera_color_optical_frame" +[INFO] [1733935073.637084]: received mask_obs with header seq: 21574 +stamp: + secs: 1733935073 + nsecs: 640948057 +frame_id: '' and grasped_object_mask header seq: 502718 +stamp: + secs: 1733935073 + nsecs: 402419806 +frame_id: "camera_color_optical_frame" +[INFO] [1733935073.884840]: received mask_obs with header seq: 21575 +stamp: + secs: 1733935073 + nsecs: 887667417 +frame_id: '' and grasped_object_mask header seq: 502725 +stamp: + secs: 1733935073 + nsecs: 635555744 +frame_id: "camera_color_optical_frame" +[INFO] [1733935074.136711]: received mask_obs with header seq: 21576 +stamp: + secs: 1733935074 + nsecs: 140179872 +frame_id: '' and grasped_object_mask header seq: 502733 +stamp: + secs: 1733935073 + nsecs: 902000904 +frame_id: "camera_color_optical_frame" +[INFO] [1733935074.389487]: received mask_obs with header seq: 21577 +stamp: + secs: 1733935074 + nsecs: 393615961 +frame_id: '' and grasped_object_mask header seq: 502741 +stamp: + secs: 1733935074 + nsecs: 168445826 +frame_id: "camera_color_optical_frame" +[INFO] [1733935074.630962]: received mask_obs with header seq: 21578 +stamp: + secs: 1733935074 + nsecs: 635043859 +frame_id: '' and grasped_object_mask header seq: 502748 +stamp: + secs: 1733935074 + nsecs: 401585340 +frame_id: "camera_color_optical_frame" +[INFO] [1733935074.898672]: received mask_obs with header seq: 21579 +stamp: + secs: 1733935074 + nsecs: 902758359 +frame_id: '' and grasped_object_mask header seq: 502756 +stamp: + secs: 1733935074 + nsecs: 668031216 +frame_id: "camera_color_optical_frame" +[INFO] [1733935075.147496]: received mask_obs with header seq: 21580 +stamp: + secs: 1733935075 + nsecs: 151490449 +frame_id: '' and grasped_object_mask header seq: 502763 +stamp: + secs: 1733935074 + nsecs: 901174307 +frame_id: "camera_color_optical_frame" +[INFO] [1733935075.401547]: received mask_obs with header seq: 21581 +stamp: + secs: 1733935075 + nsecs: 405486822 +frame_id: '' and grasped_object_mask header seq: 502771 +stamp: + secs: 1733935075 + nsecs: 167622805 +frame_id: "camera_color_optical_frame" +[INFO] [1733935075.647765]: received mask_obs with header seq: 21582 +stamp: + secs: 1733935075 + nsecs: 651572465 +frame_id: '' and grasped_object_mask header seq: 502779 +stamp: + secs: 1733935075 + nsecs: 434071541 +frame_id: "camera_color_optical_frame" +[INFO] [1733935075.902525]: received mask_obs with header seq: 21583 +stamp: + secs: 1733935075 + nsecs: 906539678 +frame_id: '' and grasped_object_mask header seq: 502786 +stamp: + secs: 1733935075 + nsecs: 667219400 +frame_id: "camera_color_optical_frame" +[INFO] [1733935076.183642]: received mask_obs with header seq: 21584 +stamp: + secs: 1733935076 + nsecs: 187725782 +frame_id: '' and grasped_object_mask header seq: 502794 +stamp: + secs: 1733935075 + nsecs: 933681250 +frame_id: "camera_color_optical_frame" +[INFO] [1733935076.291440]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1733935076.445217]: received mask_obs with header seq: 21585 +stamp: + secs: 1733935076 + nsecs: 449339389 +frame_id: '' and grasped_object_mask header seq: 502803 +stamp: + secs: 1733935076 + nsecs: 233451843 +frame_id: "camera_color_optical_frame" +[INFO] [1733935077.081062]: Stored episode 1. +[INFO] [1733935077.083163]: Saving video... +[INFO] [1733935077.278119]: Video saved! +Starting episode 1 +[INFO] [1733935077.291173]: resetting environment +[INFO] [1733935077.300396]: cleared current plan +[INFO] [1733935077.300795]: moving to home +[INFO] [1733935080.205487]: reached home +[INFO] [1733935080.206273]: reset action history +[INFO] [1733935082.918001]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1733935087.740016]: resetting environment +[INFO] [1733935087.750751]: cleared current plan +[INFO] [1733935087.751023]: moving to home +[INFO] [1733935088.752571]: reached home +[INFO] [1733935088.753044]: reset action history +[INFO] [1733935091.459991]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/episode_rosbags/episode_1_2024-12-11-11-38-11.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-743294.log +started roslaunch server http://158.130.50.37:46819/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [743771] +process[print_text-5]: started with pid [743772] +process[rosbag_record-6]: started with pid [743773] +started bagging! +[INFO] [1733935093.177025]: received mask_obs with header seq: 21586 +stamp: + secs: 1733935093 + nsecs: 179354429 +frame_id: '' and grasped_object_mask header seq: 503306 +stamp: + secs: 1733935092 + nsecs: 988657475 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 100994785 for nearest: 1733935093289245565 - target: 1733935093188250780 at idx 30 +[INFO] [1733935093.406533]: received mask_obs with header seq: 21587 +stamp: + secs: 1733935093 + nsecs: 409875631 +frame_id: '' and grasped_object_mask header seq: 503312 +stamp: + secs: 1733935093 + nsecs: 188250780 +frame_id: "camera_color_optical_frame" +[INFO] [1733935093.638938]: received mask_obs with header seq: 21588 +stamp: + secs: 1733935093 + nsecs: 642552137 +frame_id: '' and grasped_object_mask header seq: 503319 +stamp: + secs: 1733935093 + nsecs: 421094894 +frame_id: "camera_color_optical_frame" +[INFO] [1733935093.894248]: received mask_obs with header seq: 21589 +stamp: + secs: 1733935093 + nsecs: 897981882 +frame_id: '' and grasped_object_mask header seq: 503326 +stamp: + secs: 1733935093 + nsecs: 653929710 +frame_id: "camera_color_optical_frame" +[INFO] [1733935094.123034]: received mask_obs with header seq: 21590 +stamp: + secs: 1733935094 + nsecs: 125929117 +frame_id: '' and grasped_object_mask header seq: 503333 +stamp: + secs: 1733935093 + nsecs: 886817932 +frame_id: "camera_color_optical_frame" +[INFO] [1733935094.370382]: received mask_obs with header seq: 21591 +stamp: + secs: 1733935094 + nsecs: 371726989 +frame_id: '' and grasped_object_mask header seq: 503341 +stamp: + secs: 1733935094 + nsecs: 152956009 +frame_id: "camera_color_optical_frame" +[INFO] [1733935094.611299]: received mask_obs with header seq: 21592 +stamp: + secs: 1733935094 + nsecs: 610927820 +frame_id: '' and grasped_object_mask header seq: 503348 +stamp: + secs: 1733935094 + nsecs: 385810852 +frame_id: "camera_color_optical_frame" +[INFO] [1733935094.850490]: received mask_obs with header seq: 21593 +stamp: + secs: 1733935094 + nsecs: 854625225 +frame_id: '' and grasped_object_mask header seq: 503355 +stamp: + secs: 1733935094 + nsecs: 618649960 +frame_id: "camera_color_optical_frame" +[INFO] [1733935095.093350]: received mask_obs with header seq: 21594 +stamp: + secs: 1733935095 + nsecs: 96895456 +frame_id: '' and grasped_object_mask header seq: 503362 +stamp: + secs: 1733935094 + nsecs: 851536751 +frame_id: "camera_color_optical_frame" +[INFO] [1733935095.342629]: received mask_obs with header seq: 21595 +stamp: + secs: 1733935095 + nsecs: 344056129 +frame_id: '' and grasped_object_mask header seq: 503370 +stamp: + secs: 1733935095 + nsecs: 117679119 +frame_id: "camera_color_optical_frame" +[INFO] [1733935095.579948]: received mask_obs with header seq: 21596 +stamp: + secs: 1733935095 + nsecs: 583214998 +frame_id: '' and grasped_object_mask header seq: 503377 +stamp: + secs: 1733935095 + nsecs: 350537539 +frame_id: "camera_color_optical_frame" +[INFO] [1733935095.811045]: received mask_obs with header seq: 21597 +stamp: + secs: 1733935095 + nsecs: 814977169 +frame_id: '' and grasped_object_mask header seq: 503384 +stamp: + secs: 1733935095 + nsecs: 583380938 +frame_id: "camera_color_optical_frame" +[INFO] [1733935096.076857]: received mask_obs with header seq: 21598 +stamp: + secs: 1733935096 + nsecs: 76883792 +frame_id: '' and grasped_object_mask header seq: 503392 +stamp: + secs: 1733935095 + nsecs: 849521637 +frame_id: "camera_color_optical_frame" +[INFO] [1733935096.341752]: received mask_obs with header seq: 21599 +stamp: + secs: 1733935096 + nsecs: 343281507 +frame_id: '' and grasped_object_mask header seq: 503400 +stamp: + secs: 1733935096 + nsecs: 115652800 +frame_id: "camera_color_optical_frame" +[INFO] [1733935096.607314]: received mask_obs with header seq: 21600 +stamp: + secs: 1733935096 + nsecs: 609924077 +frame_id: '' and grasped_object_mask header seq: 503408 +stamp: + secs: 1733935096 + nsecs: 381763697 +frame_id: "camera_color_optical_frame" +[INFO] [1733935096.860352]: received mask_obs with header seq: 21601 +stamp: + secs: 1733935096 + nsecs: 864387273 +frame_id: '' and grasped_object_mask header seq: 503415 +stamp: + secs: 1733935096 + nsecs: 614592314 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 108471971 for nearest: 1733935096989210468 - target: 1733935096880738497 at idx 81 +[INFO] [1733935097.129432]: received mask_obs with header seq: 21602 +stamp: + secs: 1733935097 + nsecs: 133330583 +frame_id: '' and grasped_object_mask header seq: 503423 +stamp: + secs: 1733935096 + nsecs: 880738497 +frame_id: "camera_color_optical_frame" +[INFO] [1733935097.355486]: received mask_obs with header seq: 21603 +stamp: + secs: 1733935097 + nsecs: 359109640 +frame_id: '' and grasped_object_mask header seq: 503430 +stamp: + secs: 1733935097 + nsecs: 113603354 +frame_id: "camera_color_optical_frame" +[INFO] [1733935097.597824]: received mask_obs with header seq: 21604 +stamp: + secs: 1733935097 + nsecs: 601511955 +frame_id: '' and grasped_object_mask header seq: 503437 +stamp: + secs: 1733935097 + nsecs: 346451998 +frame_id: "camera_color_optical_frame" +[INFO] [1733935097.855445]: received mask_obs with header seq: 21605 +stamp: + secs: 1733935097 + nsecs: 859459400 +frame_id: '' and grasped_object_mask header seq: 503445 +stamp: + secs: 1733935097 + nsecs: 612544298 +frame_id: "camera_color_optical_frame" +[INFO] [1733935098.089305]: received mask_obs with header seq: 21606 +stamp: + secs: 1733935098 + nsecs: 93394994 +frame_id: '' and grasped_object_mask header seq: 503452 +stamp: + secs: 1733935097 + nsecs: 847348452 +frame_id: "camera_color_optical_frame" +[INFO] [1733935098.314953]: received mask_obs with header seq: 21607 +stamp: + secs: 1733935098 + nsecs: 318479537 +frame_id: '' and grasped_object_mask header seq: 503459 +stamp: + secs: 1733935098 + nsecs: 82991838 +frame_id: "camera_color_optical_frame" +[INFO] [1733935098.558080]: received mask_obs with header seq: 21608 +stamp: + secs: 1733935098 + nsecs: 562053918 +frame_id: '' and grasped_object_mask header seq: 503466 +stamp: + secs: 1733935098 + nsecs: 318738937 +frame_id: "camera_color_optical_frame" +[INFO] [1733935098.784512]: received mask_obs with header seq: 21609 +stamp: + secs: 1733935098 + nsecs: 787675142 +frame_id: '' and grasped_object_mask header seq: 503473 +stamp: + secs: 1733935098 + nsecs: 554588318 +frame_id: "camera_color_optical_frame" +[INFO] [1733935099.058629]: received mask_obs with header seq: 21610 +stamp: + secs: 1733935099 + nsecs: 57174205 +frame_id: '' and grasped_object_mask header seq: 503481 +stamp: + secs: 1733935098 + nsecs: 822340727 +frame_id: "camera_color_optical_frame" +[INFO] [1733935099.309611]: received mask_obs with header seq: 21611 +stamp: + secs: 1733935099 + nsecs: 313660144 +frame_id: '' and grasped_object_mask header seq: 503489 +stamp: + secs: 1733935099 + nsecs: 88181973 +frame_id: "camera_color_optical_frame" +[INFO] [1733935099.582594]: received mask_obs with header seq: 21612 +stamp: + secs: 1733935099 + nsecs: 586291551 +frame_id: '' and grasped_object_mask header seq: 503496 +stamp: + secs: 1733935099 + nsecs: 320759773 +frame_id: "camera_color_optical_frame" +[INFO] [1733935099.827436]: received mask_obs with header seq: 21613 +stamp: + secs: 1733935099 + nsecs: 831567287 +frame_id: '' and grasped_object_mask header seq: 503504 +stamp: + secs: 1733935099 + nsecs: 586524248 +frame_id: "camera_color_optical_frame" +[INFO] [1733935100.073479]: received mask_obs with header seq: 21614 +stamp: + secs: 1733935100 + nsecs: 77468156 +frame_id: '' and grasped_object_mask header seq: 503512 +stamp: + secs: 1733935099 + nsecs: 853337288 +frame_id: "camera_color_optical_frame" +[INFO] [1733935100.316880]: received mask_obs with header seq: 21615 +stamp: + secs: 1733935100 + nsecs: 320971488 +frame_id: '' and grasped_object_mask header seq: 503519 +stamp: + secs: 1733935100 + nsecs: 87557793 +frame_id: "camera_color_optical_frame" +[INFO] [1733935100.564981]: received mask_obs with header seq: 21616 +stamp: + secs: 1733935100 + nsecs: 568855285 +frame_id: '' and grasped_object_mask header seq: 503526 +stamp: + secs: 1733935100 + nsecs: 321882248 +frame_id: "camera_color_optical_frame" +[INFO] [1733935100.821585]: received mask_obs with header seq: 21617 +stamp: + secs: 1733935100 + nsecs: 825643539 +frame_id: '' and grasped_object_mask header seq: 503534 +stamp: + secs: 1733935100 + nsecs: 589808702 +frame_id: "camera_color_optical_frame" +[INFO] [1733935101.088067]: received mask_obs with header seq: 21618 +stamp: + secs: 1733935101 + nsecs: 92089891 +frame_id: '' and grasped_object_mask header seq: 503542 +stamp: + secs: 1733935100 + nsecs: 856764078 +frame_id: "camera_color_optical_frame" +[INFO] [1733935101.341517]: received mask_obs with header seq: 21619 +stamp: + secs: 1733935101 + nsecs: 344350814 +frame_id: '' and grasped_object_mask header seq: 503550 +stamp: + secs: 1733935101 + nsecs: 122892141 +frame_id: "camera_color_optical_frame" +[INFO] [1733935101.592115]: received mask_obs with header seq: 21620 +stamp: + secs: 1733935101 + nsecs: 596188068 +frame_id: '' and grasped_object_mask header seq: 503557 +stamp: + secs: 1733935101 + nsecs: 355737209 +frame_id: "camera_color_optical_frame" +[INFO] [1733935101.855077]: received mask_obs with header seq: 21621 +stamp: + secs: 1733935101 + nsecs: 858713865 +frame_id: '' and grasped_object_mask header seq: 503565 +stamp: + secs: 1733935101 + nsecs: 621825695 +frame_id: "camera_color_optical_frame" +[INFO] [1733935102.106812]: received mask_obs with header seq: 21622 +stamp: + secs: 1733935102 + nsecs: 110451459 +frame_id: '' and grasped_object_mask header seq: 503573 +stamp: + secs: 1733935101 + nsecs: 887951136 +frame_id: "camera_color_optical_frame" +[INFO] [1733935102.364200]: received mask_obs with header seq: 21623 +stamp: + secs: 1733935102 + nsecs: 367934465 +frame_id: '' and grasped_object_mask header seq: 503580 +stamp: + secs: 1733935102 + nsecs: 120816946 +frame_id: "camera_color_optical_frame" +[INFO] [1733935102.644447]: received mask_obs with header seq: 21624 +stamp: + secs: 1733935102 + nsecs: 647905111 +frame_id: '' and grasped_object_mask header seq: 503588 +stamp: + secs: 1733935102 + nsecs: 386929035 +frame_id: "camera_color_optical_frame" +[INFO] [1733935102.889945]: received mask_obs with header seq: 21625 +stamp: + secs: 1733935102 + nsecs: 893915414 +frame_id: '' and grasped_object_mask header seq: 503596 +stamp: + secs: 1733935102 + nsecs: 653021097 +frame_id: "camera_color_optical_frame" +[INFO] [1733935103.145015]: received mask_obs with header seq: 21626 +stamp: + secs: 1733935103 + nsecs: 148851394 +frame_id: '' and grasped_object_mask header seq: 503604 +stamp: + secs: 1733935102 + nsecs: 919148684 +frame_id: "camera_color_optical_frame" +[INFO] [1733935103.380231]: received mask_obs with header seq: 21627 +stamp: + secs: 1733935103 + nsecs: 383736371 +frame_id: '' and grasped_object_mask header seq: 503611 +stamp: + secs: 1733935103 + nsecs: 152004719 +frame_id: "camera_color_optical_frame" +[INFO] [1733935103.608634]: received mask_obs with header seq: 21628 +stamp: + secs: 1733935103 + nsecs: 612436056 +frame_id: '' and grasped_object_mask header seq: 503618 +stamp: + secs: 1733935103 + nsecs: 384843349 +frame_id: "camera_color_optical_frame" +[INFO] [1733935103.854523]: received mask_obs with header seq: 21629 +stamp: + secs: 1733935103 + nsecs: 857461452 +frame_id: '' and grasped_object_mask header seq: 503624 +stamp: + secs: 1733935103 + nsecs: 584406376 +frame_id: "camera_color_optical_frame" +[INFO] [1733935104.107121]: received mask_obs with header seq: 21630 +stamp: + secs: 1733935104 + nsecs: 110012531 +frame_id: '' and grasped_object_mask header seq: 503633 +stamp: + secs: 1733935103 + nsecs: 883793831 +frame_id: "camera_color_optical_frame" +[INFO] [1733935104.358541]: received mask_obs with header seq: 21631 +stamp: + secs: 1733935104 + nsecs: 362331390 +frame_id: '' and grasped_object_mask header seq: 503641 +stamp: + secs: 1733935104 + nsecs: 149930239 +frame_id: "camera_color_optical_frame" +[INFO] [1733935104.605721]: received mask_obs with header seq: 21632 +stamp: + secs: 1733935104 + nsecs: 609737873 +frame_id: '' and grasped_object_mask header seq: 503648 +stamp: + secs: 1733935104 + nsecs: 382783175 +frame_id: "camera_color_optical_frame" +[INFO] [1733935104.866796]: received mask_obs with header seq: 21633 +stamp: + secs: 1733935104 + nsecs: 870595693 +frame_id: '' and grasped_object_mask header seq: 503656 +stamp: + secs: 1733935104 + nsecs: 648880959 +frame_id: "camera_color_optical_frame" +[INFO] [1733935105.114484]: received mask_obs with header seq: 21634 +stamp: + secs: 1733935105 + nsecs: 118558645 +frame_id: '' and grasped_object_mask header seq: 503663 +stamp: + secs: 1733935104 + nsecs: 881753922 +frame_id: "camera_color_optical_frame" +[INFO] [1733935105.359443]: received mask_obs with header seq: 21635 +stamp: + secs: 1733935105 + nsecs: 363569974 +frame_id: '' and grasped_object_mask header seq: 503671 +stamp: + secs: 1733935105 + nsecs: 147891521 +frame_id: "camera_color_optical_frame" +[INFO] [1733935105.601279]: received mask_obs with header seq: 21636 +stamp: + secs: 1733935105 + nsecs: 604927301 +frame_id: '' and grasped_object_mask header seq: 503678 +stamp: + secs: 1733935105 + nsecs: 380745649 +frame_id: "camera_color_optical_frame" +[INFO] [1733935105.836188]: received mask_obs with header seq: 21637 +stamp: + secs: 1733935105 + nsecs: 839296579 +frame_id: '' and grasped_object_mask header seq: 503685 +stamp: + secs: 1733935105 + nsecs: 613583088 +frame_id: "camera_color_optical_frame" +[INFO] [1733935106.106758]: received mask_obs with header seq: 21638 +stamp: + secs: 1733935106 + nsecs: 110646009 +frame_id: '' and grasped_object_mask header seq: 503693 +stamp: + secs: 1733935105 + nsecs: 879708767 +frame_id: "camera_color_optical_frame" +[INFO] [1733935106.340141]: received mask_obs with header seq: 21639 +stamp: + secs: 1733935106 + nsecs: 344187974 +frame_id: '' and grasped_object_mask header seq: 503700 +stamp: + secs: 1733935106 + nsecs: 112567663 +frame_id: "camera_color_optical_frame" +[INFO] [1733935106.593508]: received mask_obs with header seq: 21640 +stamp: + secs: 1733935106 + nsecs: 594310045 +frame_id: '' and grasped_object_mask header seq: 503708 +stamp: + secs: 1733935106 + nsecs: 378673553 +frame_id: "camera_color_optical_frame" +[INFO] [1733935106.832268]: received mask_obs with header seq: 21641 +stamp: + secs: 1733935106 + nsecs: 835799694 +frame_id: '' and grasped_object_mask header seq: 503715 +stamp: + secs: 1733935106 + nsecs: 611497641 +frame_id: "camera_color_optical_frame" +[INFO] [1733935107.060644]: received mask_obs with header seq: 21642 +stamp: + secs: 1733935107 + nsecs: 64822435 +frame_id: '' and grasped_object_mask header seq: 503722 +stamp: + secs: 1733935106 + nsecs: 844351053 +frame_id: "camera_color_optical_frame" +[INFO] [1733935107.309254]: received mask_obs with header seq: 21643 +stamp: + secs: 1733935107 + nsecs: 313229560 +frame_id: '' and grasped_object_mask header seq: 503729 +stamp: + secs: 1733935107 + nsecs: 77214241 +frame_id: "camera_color_optical_frame" +[INFO] [1733935107.566648]: received mask_obs with header seq: 21644 +stamp: + secs: 1733935107 + nsecs: 570574283 +frame_id: '' and grasped_object_mask header seq: 503737 +stamp: + secs: 1733935107 + nsecs: 343322992 +frame_id: "camera_color_optical_frame" +[INFO] [1733935107.837627]: received mask_obs with header seq: 21645 +stamp: + secs: 1733935107 + nsecs: 841741085 +frame_id: '' and grasped_object_mask header seq: 503745 +stamp: + secs: 1733935107 + nsecs: 609410763 +frame_id: "camera_color_optical_frame" +[INFO] [1733935108.091443]: received mask_obs with header seq: 21646 +stamp: + secs: 1733935108 + nsecs: 94268321 +frame_id: '' and grasped_object_mask header seq: 503753 +stamp: + secs: 1733935107 + nsecs: 875537872 +frame_id: "camera_color_optical_frame" +[INFO] [1733935108.365844]: received mask_obs with header seq: 21647 +stamp: + secs: 1733935108 + nsecs: 369434356 +frame_id: '' and grasped_object_mask header seq: 503761 +stamp: + secs: 1733935108 + nsecs: 141673803 +frame_id: "camera_color_optical_frame" +[INFO] [1733935108.617560]: received mask_obs with header seq: 21648 +stamp: + secs: 1733935108 + nsecs: 621635675 +frame_id: '' and grasped_object_mask header seq: 503768 +stamp: + secs: 1733935108 + nsecs: 374524832 +frame_id: "camera_color_optical_frame" +[INFO] [1733935108.873827]: received mask_obs with header seq: 21649 +stamp: + secs: 1733935108 + nsecs: 876623153 +frame_id: '' and grasped_object_mask header seq: 503776 +stamp: + secs: 1733935108 + nsecs: 640621424 +frame_id: "camera_color_optical_frame" +[INFO] [1733935109.124899]: received mask_obs with header seq: 21650 +stamp: + secs: 1733935109 + nsecs: 127112865 +frame_id: '' and grasped_object_mask header seq: 503784 +stamp: + secs: 1733935108 + nsecs: 906759739 +frame_id: "camera_color_optical_frame" +[INFO] [1733935109.390679]: received mask_obs with header seq: 21651 +stamp: + secs: 1733935109 + nsecs: 392376661 +frame_id: '' and grasped_object_mask header seq: 503791 +stamp: + secs: 1733935109 + nsecs: 139626026 +frame_id: "camera_color_optical_frame" +[INFO] [1733935109.625639]: received mask_obs with header seq: 21652 +stamp: + secs: 1733935109 + nsecs: 629604339 +frame_id: '' and grasped_object_mask header seq: 503799 +stamp: + secs: 1733935109 + nsecs: 405740738 +frame_id: "camera_color_optical_frame" +[INFO] [1733935109.897466]: received mask_obs with header seq: 21653 +stamp: + secs: 1733935109 + nsecs: 901027917 +frame_id: '' and grasped_object_mask header seq: 503807 +stamp: + secs: 1733935109 + nsecs: 671833754 +frame_id: "camera_color_optical_frame" +[INFO] [1733935110.147359]: received mask_obs with header seq: 21654 +stamp: + secs: 1733935110 + nsecs: 151443958 +frame_id: '' and grasped_object_mask header seq: 503814 +stamp: + secs: 1733935109 + nsecs: 904714823 +frame_id: "camera_color_optical_frame" +[INFO] [1733935110.391440]: received mask_obs with header seq: 21655 +stamp: + secs: 1733935110 + nsecs: 395336627 +frame_id: '' and grasped_object_mask header seq: 503822 +stamp: + secs: 1733935110 + nsecs: 170854330 +frame_id: "camera_color_optical_frame" +[INFO] [1733935110.657335]: received mask_obs with header seq: 21656 +stamp: + secs: 1733935110 + nsecs: 659604787 +frame_id: '' and grasped_object_mask header seq: 503830 +stamp: + secs: 1733935110 + nsecs: 436972380 +frame_id: "camera_color_optical_frame" +[INFO] [1733935110.904506]: received mask_obs with header seq: 21657 +stamp: + secs: 1733935110 + nsecs: 908251762 +frame_id: '' and grasped_object_mask header seq: 503837 +stamp: + secs: 1733935110 + nsecs: 669809580 +frame_id: "camera_color_optical_frame" +[INFO] [1733935111.154010]: received mask_obs with header seq: 21658 +stamp: + secs: 1733935111 + nsecs: 156890630 +frame_id: '' and grasped_object_mask header seq: 503845 +stamp: + secs: 1733935110 + nsecs: 935960531 +frame_id: "camera_color_optical_frame" +[INFO] [1733935111.400201]: received mask_obs with header seq: 21659 +stamp: + secs: 1733935111 + nsecs: 404329776 +frame_id: '' and grasped_object_mask header seq: 503852 +stamp: + secs: 1733935111 + nsecs: 168831348 +frame_id: "camera_color_optical_frame" +[INFO] [1733935111.647763]: received mask_obs with header seq: 21660 +stamp: + secs: 1733935111 + nsecs: 651881933 +frame_id: '' and grasped_object_mask header seq: 503859 +stamp: + secs: 1733935111 + nsecs: 401685238 +frame_id: "camera_color_optical_frame" +[INFO] [1733935111.893348]: received mask_obs with header seq: 21661 +stamp: + secs: 1733935111 + nsecs: 896712779 +frame_id: '' and grasped_object_mask header seq: 503866 +stamp: + secs: 1733935111 + nsecs: 634524822 +frame_id: "camera_color_optical_frame" +[INFO] [1733935112.126202]: received mask_obs with header seq: 21662 +stamp: + secs: 1733935112 + nsecs: 130305051 +frame_id: '' and grasped_object_mask header seq: 503874 +stamp: + secs: 1733935111 + nsecs: 900663853 +frame_id: "camera_color_optical_frame" +[INFO] [1733935112.389923]: received mask_obs with header seq: 21663 +stamp: + secs: 1733935112 + nsecs: 393917322 +frame_id: '' and grasped_object_mask header seq: 503882 +stamp: + secs: 1733935112 + nsecs: 166796446 +frame_id: "camera_color_optical_frame" +[INFO] [1733935112.658146]: received mask_obs with header seq: 21664 +stamp: + secs: 1733935112 + nsecs: 661280870 +frame_id: '' and grasped_object_mask header seq: 503890 +stamp: + secs: 1733935112 + nsecs: 432909250 +frame_id: "camera_color_optical_frame" +[INFO] [1733935112.901284]: received mask_obs with header seq: 21665 +stamp: + secs: 1733935112 + nsecs: 905409574 +frame_id: '' and grasped_object_mask header seq: 503897 +stamp: + secs: 1733935112 + nsecs: 665739536 +frame_id: "camera_color_optical_frame" +[INFO] [1733935113.154137]: received mask_obs with header seq: 21666 +stamp: + secs: 1733935113 + nsecs: 157136440 +frame_id: '' and grasped_object_mask header seq: 503905 +stamp: + secs: 1733935112 + nsecs: 931884527 +frame_id: "camera_color_optical_frame" +[INFO] [1733935113.413823]: received mask_obs with header seq: 21667 +stamp: + secs: 1733935113 + nsecs: 417340517 +frame_id: '' and grasped_object_mask header seq: 503912 +stamp: + secs: 1733935113 + nsecs: 164750576 +frame_id: "camera_color_optical_frame" +[INFO] [1733935113.658682]: received mask_obs with header seq: 21668 +stamp: + secs: 1733935113 + nsecs: 662591218 +frame_id: '' and grasped_object_mask header seq: 503920 +stamp: + secs: 1733935113 + nsecs: 430862904 +frame_id: "camera_color_optical_frame" +[INFO] [1733935113.927479]: received mask_obs with header seq: 21669 +stamp: + secs: 1733935113 + nsecs: 928974151 +frame_id: '' and grasped_object_mask header seq: 503928 +stamp: + secs: 1733935113 + nsecs: 697028399 +frame_id: "camera_color_optical_frame" +[INFO] [1733935114.156650]: received mask_obs with header seq: 21670 +stamp: + secs: 1733935114 + nsecs: 160588026 +frame_id: '' and grasped_object_mask header seq: 503935 +stamp: + secs: 1733935113 + nsecs: 931422234 +frame_id: "camera_color_optical_frame" +[INFO] [1733935114.400995]: received mask_obs with header seq: 21671 +stamp: + secs: 1733935114 + nsecs: 405052185 +frame_id: '' and grasped_object_mask header seq: 503942 +stamp: + secs: 1733935114 + nsecs: 165935993 +frame_id: "camera_color_optical_frame" +[INFO] [1733935114.655312]: received mask_obs with header seq: 21672 +stamp: + secs: 1733935114 + nsecs: 659463644 +frame_id: '' and grasped_object_mask header seq: 503950 +stamp: + secs: 1733935114 + nsecs: 434097052 +frame_id: "camera_color_optical_frame" +[INFO] [1733935114.904258]: received mask_obs with header seq: 21673 +stamp: + secs: 1733935114 + nsecs: 908096313 +frame_id: '' and grasped_object_mask header seq: 503957 +stamp: + secs: 1733935114 + nsecs: 668866396 +frame_id: "camera_color_optical_frame" +[INFO] [1733935115.171147]: received mask_obs with header seq: 21674 +stamp: + secs: 1733935115 + nsecs: 175046682 +frame_id: '' and grasped_object_mask header seq: 503965 +stamp: + secs: 1733935114 + nsecs: 935487509 +frame_id: "camera_color_optical_frame" +[INFO] [1733935115.454461]: received mask_obs with header seq: 21675 +stamp: + secs: 1733935115 + nsecs: 453802347 +frame_id: '' and grasped_object_mask header seq: 503973 +stamp: + secs: 1733935115 + nsecs: 201946735 +frame_id: "camera_color_optical_frame" +[INFO] [1733935115.711265]: received mask_obs with header seq: 21676 +stamp: + secs: 1733935115 + nsecs: 715339660 +frame_id: '' and grasped_object_mask header seq: 503981 +stamp: + secs: 1733935115 + nsecs: 468406916 +frame_id: "camera_color_optical_frame" +[INFO] [1733935115.952562]: received mask_obs with header seq: 21677 +stamp: + secs: 1733935115 + nsecs: 955738782 +frame_id: '' and grasped_object_mask header seq: 503989 +stamp: + secs: 1733935115 + nsecs: 734869480 +frame_id: "camera_color_optical_frame" +[INFO] [1733935116.187355]: received mask_obs with header seq: 21678 +stamp: + secs: 1733935116 + nsecs: 188242912 +frame_id: '' and grasped_object_mask header seq: 503996 +stamp: + secs: 1733935115 + nsecs: 968026638 +frame_id: "camera_color_optical_frame" +[INFO] [1733935116.450840]: received mask_obs with header seq: 21679 +stamp: + secs: 1733935116 + nsecs: 453866243 +frame_id: '' and grasped_object_mask header seq: 504004 +stamp: + secs: 1733935116 + nsecs: 234494686 +frame_id: "camera_color_optical_frame" +[INFO] [1733935116.709226]: received mask_obs with header seq: 21680 +stamp: + secs: 1733935116 + nsecs: 713304281 +frame_id: '' and grasped_object_mask header seq: 504011 +stamp: + secs: 1733935116 + nsecs: 467653513 +frame_id: "camera_color_optical_frame" +[INFO] [1733935116.956172]: received mask_obs with header seq: 21681 +stamp: + secs: 1733935116 + nsecs: 960238218 +frame_id: '' and grasped_object_mask header seq: 504019 +stamp: + secs: 1733935116 + nsecs: 734120607 +frame_id: "camera_color_optical_frame" +[INFO] [1733935117.200190]: received mask_obs with header seq: 21682 +stamp: + secs: 1733935117 + nsecs: 203858137 +frame_id: '' and grasped_object_mask header seq: 504026 +stamp: + secs: 1733935116 + nsecs: 967265368 +frame_id: "camera_color_optical_frame" +[INFO] [1733935117.450025]: received mask_obs with header seq: 21683 +stamp: + secs: 1733935117 + nsecs: 453884124 +frame_id: '' and grasped_object_mask header seq: 504034 +stamp: + secs: 1733935117 + nsecs: 233716726 +frame_id: "camera_color_optical_frame" +[INFO] [1733935117.697202]: received mask_obs with header seq: 21684 +stamp: + secs: 1733935117 + nsecs: 700025320 +frame_id: '' and grasped_object_mask header seq: 504041 +stamp: + secs: 1733935117 + nsecs: 466862202 +frame_id: "camera_color_optical_frame" +[INFO] [1733935117.947303]: received mask_obs with header seq: 21685 +stamp: + secs: 1733935117 + nsecs: 951388359 +frame_id: '' and grasped_object_mask header seq: 504048 +stamp: + secs: 1733935117 + nsecs: 700007677 +frame_id: "camera_color_optical_frame" +[INFO] [1733935118.187882]: received mask_obs with header seq: 21686 +stamp: + secs: 1733935118 + nsecs: 191961050 +frame_id: '' and grasped_object_mask header seq: 504056 +stamp: + secs: 1733935117 + nsecs: 966464758 +frame_id: "camera_color_optical_frame" +[INFO] [1733935118.449641]: received mask_obs with header seq: 21687 +stamp: + secs: 1733935118 + nsecs: 453249454 +frame_id: '' and grasped_object_mask header seq: 504064 +stamp: + secs: 1733935118 + nsecs: 232922554 +frame_id: "camera_color_optical_frame" +[INFO] [1733935118.710757]: received mask_obs with header seq: 21688 +stamp: + secs: 1733935118 + nsecs: 714598894 +frame_id: '' and grasped_object_mask header seq: 504071 +stamp: + secs: 1733935118 + nsecs: 466073751 +frame_id: "camera_color_optical_frame" +[INFO] [1733935118.956817]: received mask_obs with header seq: 21689 +stamp: + secs: 1733935118 + nsecs: 960974931 +frame_id: '' and grasped_object_mask header seq: 504079 +stamp: + secs: 1733935118 + nsecs: 732530832 +frame_id: "camera_color_optical_frame" +[INFO] [1733935119.183222]: received mask_obs with header seq: 21690 +stamp: + secs: 1733935119 + nsecs: 186753988 +frame_id: '' and grasped_object_mask header seq: 504086 +stamp: + secs: 1733935118 + nsecs: 965674400 +frame_id: "camera_color_optical_frame" +[INFO] [1733935119.429135]: received mask_obs with header seq: 21691 +stamp: + secs: 1733935119 + nsecs: 433315515 +frame_id: '' and grasped_object_mask header seq: 504093 +stamp: + secs: 1733935119 + nsecs: 198817492 +frame_id: "camera_color_optical_frame" +[INFO] [1733935119.690381]: received mask_obs with header seq: 21692 +stamp: + secs: 1733935119 + nsecs: 694382667 +frame_id: '' and grasped_object_mask header seq: 504101 +stamp: + secs: 1733935119 + nsecs: 465266228 +frame_id: "camera_color_optical_frame" +[INFO] [1733935119.952732]: received mask_obs with header seq: 21693 +stamp: + secs: 1733935119 + nsecs: 956776857 +frame_id: '' and grasped_object_mask header seq: 504109 +stamp: + secs: 1733935119 + nsecs: 731714010 +frame_id: "camera_color_optical_frame" +[INFO] [1733935120.215466]: received mask_obs with header seq: 21694 +stamp: + secs: 1733935120 + nsecs: 217965126 +frame_id: '' and grasped_object_mask header seq: 504116 +stamp: + secs: 1733935119 + nsecs: 964854956 +frame_id: "camera_color_optical_frame" +[INFO] [1733935120.463200]: received mask_obs with header seq: 21695 +stamp: + secs: 1733935120 + nsecs: 467159509 +frame_id: '' and grasped_object_mask header seq: 504124 +stamp: + secs: 1733935120 + nsecs: 231301069 +frame_id: "camera_color_optical_frame" +[INFO] [1733935120.729716]: received mask_obs with header seq: 21696 +stamp: + secs: 1733935120 + nsecs: 733228683 +frame_id: '' and grasped_object_mask header seq: 504132 +stamp: + secs: 1733935120 + nsecs: 497746944 +frame_id: "camera_color_optical_frame" +[INFO] [1733935120.982240]: received mask_obs with header seq: 21697 +stamp: + secs: 1733935120 + nsecs: 984909534 +frame_id: '' and grasped_object_mask header seq: 504140 +stamp: + secs: 1733935120 + nsecs: 764194727 +frame_id: "camera_color_optical_frame" +[INFO] [1733935121.227345]: received mask_obs with header seq: 21698 +stamp: + secs: 1733935121 + nsecs: 231047153 +frame_id: '' and grasped_object_mask header seq: 504147 +stamp: + secs: 1733935120 + nsecs: 997343540 +frame_id: "camera_color_optical_frame" +[INFO] [1733935121.485660]: received mask_obs with header seq: 21699 +stamp: + secs: 1733935121 + nsecs: 489724159 +frame_id: '' and grasped_object_mask header seq: 504155 +stamp: + secs: 1733935121 + nsecs: 263799191 +frame_id: "camera_color_optical_frame" +[INFO] [1733935121.716814]: received mask_obs with header seq: 21700 +stamp: + secs: 1733935121 + nsecs: 720345497 +frame_id: '' and grasped_object_mask header seq: 504162 +stamp: + secs: 1733935121 + nsecs: 496948481 +frame_id: "camera_color_optical_frame" +[INFO] [1733935121.968030]: received mask_obs with header seq: 21701 +stamp: + secs: 1733935121 + nsecs: 970935344 +frame_id: '' and grasped_object_mask header seq: 504169 +stamp: + secs: 1733935121 + nsecs: 730096340 +frame_id: "camera_color_optical_frame" +[INFO] [1733935122.218353]: received mask_obs with header seq: 21702 +stamp: + secs: 1733935122 + nsecs: 222336530 +frame_id: '' and grasped_object_mask header seq: 504177 +stamp: + secs: 1733935121 + nsecs: 996540546 +frame_id: "camera_color_optical_frame" +[INFO] [1733935122.451247]: received mask_obs with header seq: 21703 +stamp: + secs: 1733935122 + nsecs: 455412149 +frame_id: '' and grasped_object_mask header seq: 504184 +stamp: + secs: 1733935122 + nsecs: 229678154 +frame_id: "camera_color_optical_frame" +[INFO] [1733935122.697322]: received mask_obs with header seq: 21704 +stamp: + secs: 1733935122 + nsecs: 700396776 +frame_id: '' and grasped_object_mask header seq: 504191 +stamp: + secs: 1733935122 + nsecs: 462816238 +frame_id: "camera_color_optical_frame" +[INFO] [1733935122.947654]: received mask_obs with header seq: 21705 +stamp: + secs: 1733935122 + nsecs: 949034214 +frame_id: '' and grasped_object_mask header seq: 504199 +stamp: + secs: 1733935122 + nsecs: 729258776 +frame_id: "camera_color_optical_frame" +[INFO] [1733935123.179087]: received mask_obs with header seq: 21706 +stamp: + secs: 1733935123 + nsecs: 183083057 +frame_id: '' and grasped_object_mask header seq: 504206 +stamp: + secs: 1733935122 + nsecs: 962396860 +frame_id: "camera_color_optical_frame" +[INFO] [1733935123.423031]: received mask_obs with header seq: 21707 +stamp: + secs: 1733935123 + nsecs: 426610708 +frame_id: '' and grasped_object_mask header seq: 504213 +stamp: + secs: 1733935123 + nsecs: 195534706 +frame_id: "camera_color_optical_frame" +[INFO] [1733935123.646553]: received mask_obs with header seq: 21708 +stamp: + secs: 1733935123 + nsecs: 648728847 +frame_id: '' and grasped_object_mask header seq: 504220 +stamp: + secs: 1733935123 + nsecs: 428670883 +frame_id: "camera_color_optical_frame" +[INFO] [1733935123.880259]: received mask_obs with header seq: 21709 +stamp: + secs: 1733935123 + nsecs: 884407281 +frame_id: '' and grasped_object_mask header seq: 504227 +stamp: + secs: 1733935123 + nsecs: 661807537 +frame_id: "camera_color_optical_frame" +[INFO] [1733935124.114050]: received mask_obs with header seq: 21710 +stamp: + secs: 1733935124 + nsecs: 117954254 +frame_id: '' and grasped_object_mask header seq: 504234 +stamp: + secs: 1733935123 + nsecs: 894947052 +frame_id: "camera_color_optical_frame" +[INFO] [1733935124.377072]: received mask_obs with header seq: 21711 +stamp: + secs: 1733935124 + nsecs: 380333423 +frame_id: '' and grasped_object_mask header seq: 504242 +stamp: + secs: 1733935124 + nsecs: 161391497 +frame_id: "camera_color_optical_frame" +[INFO] [1733935124.617695]: received mask_obs with header seq: 21712 +stamp: + secs: 1733935124 + nsecs: 621774435 +frame_id: '' and grasped_object_mask header seq: 504249 +stamp: + secs: 1733935124 + nsecs: 394531012 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -105172082 for nearest: 1733935124522496776 - target: 1733935124627668858 at idx 249 +[INFO] [1733935124.844651]: received mask_obs with header seq: 21713 +stamp: + secs: 1733935124 + nsecs: 848341941 +frame_id: '' and grasped_object_mask header seq: 504256 +stamp: + secs: 1733935124 + nsecs: 627668858 +frame_id: "camera_color_optical_frame" +[INFO] [1733935125.094492]: received mask_obs with header seq: 21714 +stamp: + secs: 1733935125 + nsecs: 97578763 +frame_id: '' and grasped_object_mask header seq: 504263 +stamp: + secs: 1733935124 + nsecs: 860810041 +frame_id: "camera_color_optical_frame" +[INFO] [1733935125.365945]: received mask_obs with header seq: 21715 +stamp: + secs: 1733935125 + nsecs: 370007753 +frame_id: '' and grasped_object_mask header seq: 504271 +stamp: + secs: 1733935125 + nsecs: 127257347 +frame_id: "camera_color_optical_frame" +[INFO] [1733935125.594828]: received mask_obs with header seq: 21716 +stamp: + secs: 1733935125 + nsecs: 598654270 +frame_id: '' and grasped_object_mask header seq: 504278 +stamp: + secs: 1733935125 + nsecs: 360399008 +frame_id: "camera_color_optical_frame" +[INFO] [1733935125.846279]: received mask_obs with header seq: 21717 +stamp: + secs: 1733935125 + nsecs: 847130298 +frame_id: '' and grasped_object_mask header seq: 504286 +stamp: + secs: 1733935125 + nsecs: 626845837 +frame_id: "camera_color_optical_frame" +[INFO] [1733935126.089357]: received mask_obs with header seq: 21718 +stamp: + secs: 1733935126 + nsecs: 93413829 +frame_id: '' and grasped_object_mask header seq: 504293 +stamp: + secs: 1733935125 + nsecs: 861184120 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -108234495 for nearest: 1733935126022544295 - target: 1733935126130778790 at idx 249 +[INFO] [1733935126.340521]: received mask_obs with header seq: 21719 +stamp: + secs: 1733935126 + nsecs: 344288349 +frame_id: '' and grasped_object_mask header seq: 504301 +stamp: + secs: 1733935126 + nsecs: 130778790 +frame_id: "camera_color_optical_frame" +[INFO] [1733935126.589922]: received mask_obs with header seq: 21720 +stamp: + secs: 1733935126 + nsecs: 593990564 +frame_id: '' and grasped_object_mask header seq: 504308 +stamp: + secs: 1733935126 + nsecs: 366801500 +frame_id: "camera_color_optical_frame" +[INFO] [1733935126.697870]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1733935126.842128]: received mask_obs with header seq: 21721 +stamp: + secs: 1733935126 + nsecs: 844366788 +frame_id: '' and grasped_object_mask header seq: 504316 +stamp: + secs: 1733935126 + nsecs: 636686325 +frame_id: "camera_color_optical_frame" +[INFO] [1733935127.455147]: Stored episode 2. +[INFO] [1733935127.461049]: Saving video... +[INFO] [1733935127.711537]: Video saved! +Starting episode 2 +[INFO] [1733935127.752792]: resetting environment +[INFO] [1733935127.795556]: cleared current plan +[INFO] [1733935127.797873]: moving to home +[INFO] [1733935130.404602]: reached home +[INFO] [1733935130.406068]: reset action history +[INFO] [1733935133.137383]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1733935161.100628]: resetting environment +[INFO] [1733935161.116476]: cleared current plan +[INFO] [1733935161.117561]: moving to home +[INFO] [1733935162.124095]: reached home +[INFO] [1733935162.125851]: reset action history +[INFO] [1733935164.853225]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/episode_rosbags/episode_2_2024-12-11-11-39-24.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-743294.log +started roslaunch server http://158.130.50.37:38991/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [743916] +process[print_text-8]: started with pid [743917] +process[rosbag_record-9]: started with pid [743918] +started bagging! +[INFO] [1733935167.592008]: received mask_obs with header seq: 21722 +stamp: + secs: 1733935167 + nsecs: 595867633 +frame_id: '' and grasped_object_mask header seq: 505540 +stamp: + secs: 1733935167 + nsecs: 393270493 +frame_id: "camera_color_optical_frame" +[INFO] [1733935167.825725]: received mask_obs with header seq: 21723 +stamp: + secs: 1733935167 + nsecs: 829886913 +frame_id: '' and grasped_object_mask header seq: 505547 +stamp: + secs: 1733935167 + nsecs: 626423359 +frame_id: "camera_color_optical_frame" +[INFO] [1733935168.057946]: received mask_obs with header seq: 21724 +stamp: + secs: 1733935168 + nsecs: 61961412 +frame_id: '' and grasped_object_mask header seq: 505553 +stamp: + secs: 1733935167 + nsecs: 826265812 +frame_id: "camera_color_optical_frame" +[INFO] [1733935168.297709]: received mask_obs with header seq: 21725 +stamp: + secs: 1733935168 + nsecs: 300969123 +frame_id: '' and grasped_object_mask header seq: 505560 +stamp: + secs: 1733935168 + nsecs: 59409618 +frame_id: "camera_color_optical_frame" +[INFO] [1733935168.534293]: received mask_obs with header seq: 21726 +stamp: + secs: 1733935168 + nsecs: 538372278 +frame_id: '' and grasped_object_mask header seq: 505568 +stamp: + secs: 1733935168 + nsecs: 325858831 +frame_id: "camera_color_optical_frame" +[INFO] [1733935168.778238]: received mask_obs with header seq: 21727 +stamp: + secs: 1733935168 + nsecs: 781946420 +frame_id: '' and grasped_object_mask header seq: 505575 +stamp: + secs: 1733935168 + nsecs: 559002399 +frame_id: "camera_color_optical_frame" +[INFO] [1733935169.022956]: received mask_obs with header seq: 21728 +stamp: + secs: 1733935169 + nsecs: 26944398 +frame_id: '' and grasped_object_mask header seq: 505582 +stamp: + secs: 1733935168 + nsecs: 792144775 +frame_id: "camera_color_optical_frame" +[INFO] [1733935169.284444]: received mask_obs with header seq: 21729 +stamp: + secs: 1733935169 + nsecs: 288121461 +frame_id: '' and grasped_object_mask header seq: 505590 +stamp: + secs: 1733935169 + nsecs: 58586121 +frame_id: "camera_color_optical_frame" +[INFO] [1733935169.522296]: received mask_obs with header seq: 21730 +stamp: + secs: 1733935169 + nsecs: 526496171 +frame_id: '' and grasped_object_mask header seq: 505597 +stamp: + secs: 1733935169 + nsecs: 291722775 +frame_id: "camera_color_optical_frame" +[INFO] [1733935169.781637]: received mask_obs with header seq: 21731 +stamp: + secs: 1733935169 + nsecs: 785579204 +frame_id: '' and grasped_object_mask header seq: 505605 +stamp: + secs: 1733935169 + nsecs: 558162928 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -102077215 for nearest: 1733935169722527058 - target: 1733935169824604273 at idx 66 +[INFO] [1733935170.052260]: received mask_obs with header seq: 21732 +stamp: + secs: 1733935170 + nsecs: 55716276 +frame_id: '' and grasped_object_mask header seq: 505613 +stamp: + secs: 1733935169 + nsecs: 824604273 +frame_id: "camera_color_optical_frame" +[INFO] [1733935170.304144]: received mask_obs with header seq: 21733 +stamp: + secs: 1733935170 + nsecs: 308179378 +frame_id: '' and grasped_object_mask header seq: 505620 +stamp: + secs: 1733935170 + nsecs: 57741880 +frame_id: "camera_color_optical_frame" +[INFO] [1733935170.554768]: received mask_obs with header seq: 21734 +stamp: + secs: 1733935170 + nsecs: 558816909 +frame_id: '' and grasped_object_mask header seq: 505628 +stamp: + secs: 1733935170 + nsecs: 324185133 +frame_id: "camera_color_optical_frame" +[INFO] [1733935170.801083]: received mask_obs with header seq: 21735 +stamp: + secs: 1733935170 + nsecs: 805284023 +frame_id: '' and grasped_object_mask header seq: 505635 +stamp: + secs: 1733935170 + nsecs: 557322979 +frame_id: "camera_color_optical_frame" +[INFO] [1733935171.056109]: received mask_obs with header seq: 21736 +stamp: + secs: 1733935171 + nsecs: 60284376 +frame_id: '' and grasped_object_mask header seq: 505643 +stamp: + secs: 1733935170 + nsecs: 823765039 +frame_id: "camera_color_optical_frame" +[INFO] [1733935171.302319]: received mask_obs with header seq: 21737 +stamp: + secs: 1733935171 + nsecs: 306073904 +frame_id: '' and grasped_object_mask header seq: 505651 +stamp: + secs: 1733935171 + nsecs: 90209484 +frame_id: "camera_color_optical_frame" +[INFO] [1733935171.559373]: received mask_obs with header seq: 21738 +stamp: + secs: 1733935171 + nsecs: 563542842 +frame_id: '' and grasped_object_mask header seq: 505658 +stamp: + secs: 1733935171 + nsecs: 323346138 +frame_id: "camera_color_optical_frame" +[INFO] [1733935171.800288]: received mask_obs with header seq: 21739 +stamp: + secs: 1733935171 + nsecs: 804290294 +frame_id: '' and grasped_object_mask header seq: 505665 +stamp: + secs: 1733935171 + nsecs: 556483507 +frame_id: "camera_color_optical_frame" +[INFO] [1733935172.055449]: received mask_obs with header seq: 21740 +stamp: + secs: 1733935172 + nsecs: 59405565 +frame_id: '' and grasped_object_mask header seq: 505673 +stamp: + secs: 1733935171 + nsecs: 822926283 +frame_id: "camera_color_optical_frame" +[INFO] [1733935172.306823]: received mask_obs with header seq: 21741 +stamp: + secs: 1733935172 + nsecs: 307937383 +frame_id: '' and grasped_object_mask header seq: 505681 +stamp: + secs: 1733935172 + nsecs: 89375496 +frame_id: "camera_color_optical_frame" +[INFO] [1733935172.568795]: received mask_obs with header seq: 21742 +stamp: + secs: 1733935172 + nsecs: 572574377 +frame_id: '' and grasped_object_mask header seq: 505688 +stamp: + secs: 1733935172 + nsecs: 322518587 +frame_id: "camera_color_optical_frame" +[INFO] [1733935172.828927]: received mask_obs with header seq: 21743 +stamp: + secs: 1733935172 + nsecs: 833022356 +frame_id: '' and grasped_object_mask header seq: 505696 +stamp: + secs: 1733935172 + nsecs: 588967323 +frame_id: "camera_color_optical_frame" +[INFO] [1733935173.081672]: received mask_obs with header seq: 21744 +stamp: + secs: 1733935173 + nsecs: 85764408 +frame_id: '' and grasped_object_mask header seq: 505704 +stamp: + secs: 1733935172 + nsecs: 855417252 +frame_id: "camera_color_optical_frame" +[INFO] [1733935173.321479]: received mask_obs with header seq: 21745 +stamp: + secs: 1733935173 + nsecs: 324987411 +frame_id: '' and grasped_object_mask header seq: 505711 +stamp: + secs: 1733935173 + nsecs: 88561296 +frame_id: "camera_color_optical_frame" +[INFO] [1733935173.562238]: received mask_obs with header seq: 21746 +stamp: + secs: 1733935173 + nsecs: 566177368 +frame_id: '' and grasped_object_mask header seq: 505718 +stamp: + secs: 1733935173 + nsecs: 321706295 +frame_id: "camera_color_optical_frame" +[INFO] [1733935173.820325]: received mask_obs with header seq: 21747 +stamp: + secs: 1733935173 + nsecs: 824367523 +frame_id: '' and grasped_object_mask header seq: 505726 +stamp: + secs: 1733935173 + nsecs: 588157177 +frame_id: "camera_color_optical_frame" +[INFO] [1733935174.063246]: received mask_obs with header seq: 21748 +stamp: + secs: 1733935174 + nsecs: 67404508 +frame_id: '' and grasped_object_mask header seq: 505733 +stamp: + secs: 1733935173 + nsecs: 821300745 +frame_id: "camera_color_optical_frame" +[INFO] [1733935174.322460]: received mask_obs with header seq: 21749 +stamp: + secs: 1733935174 + nsecs: 326498031 +frame_id: '' and grasped_object_mask header seq: 505741 +stamp: + secs: 1733935174 + nsecs: 87742329 +frame_id: "camera_color_optical_frame" +[INFO] [1733935174.583896]: received mask_obs with header seq: 21750 +stamp: + secs: 1733935174 + nsecs: 588109493 +frame_id: '' and grasped_object_mask header seq: 505749 +stamp: + secs: 1733935174 + nsecs: 354183674 +frame_id: "camera_color_optical_frame" +[INFO] [1733935174.836857]: received mask_obs with header seq: 21751 +stamp: + secs: 1733935174 + nsecs: 840474367 +frame_id: '' and grasped_object_mask header seq: 505756 +stamp: + secs: 1733935174 + nsecs: 587319851 +frame_id: "camera_color_optical_frame" +[INFO] [1733935175.077215]: received mask_obs with header seq: 21752 +stamp: + secs: 1733935175 + nsecs: 80780744 +frame_id: '' and grasped_object_mask header seq: 505764 +stamp: + secs: 1733935174 + nsecs: 853768349 +frame_id: "camera_color_optical_frame" +[INFO] [1733935175.337687]: received mask_obs with header seq: 21753 +stamp: + secs: 1733935175 + nsecs: 340527772 +frame_id: '' and grasped_object_mask header seq: 505771 +stamp: + secs: 1733935175 + nsecs: 86930037 +frame_id: "camera_color_optical_frame" +[INFO] [1733935175.586177]: received mask_obs with header seq: 21754 +stamp: + secs: 1733935175 + nsecs: 590277433 +frame_id: '' and grasped_object_mask header seq: 505779 +stamp: + secs: 1733935175 + nsecs: 353400946 +frame_id: "camera_color_optical_frame" +[INFO] [1733935175.847963]: received mask_obs with header seq: 21755 +stamp: + secs: 1733935175 + nsecs: 852076053 +frame_id: '' and grasped_object_mask header seq: 505787 +stamp: + secs: 1733935175 + nsecs: 619873285 +frame_id: "camera_color_optical_frame" +[INFO] [1733935176.102224]: received mask_obs with header seq: 21756 +stamp: + secs: 1733935176 + nsecs: 105340242 +frame_id: '' and grasped_object_mask header seq: 505794 +stamp: + secs: 1733935175 + nsecs: 853032589 +frame_id: "camera_color_optical_frame" +[INFO] [1733935176.363846]: received mask_obs with header seq: 21757 +stamp: + secs: 1733935176 + nsecs: 367784976 +frame_id: '' and grasped_object_mask header seq: 505802 +stamp: + secs: 1733935176 + nsecs: 119486094 +frame_id: "camera_color_optical_frame" +[INFO] [1733935176.616936]: received mask_obs with header seq: 21758 +stamp: + secs: 1733935176 + nsecs: 621052503 +frame_id: '' and grasped_object_mask header seq: 505810 +stamp: + secs: 1733935176 + nsecs: 385939598 +frame_id: "camera_color_optical_frame" +[INFO] [1733935176.872215]: received mask_obs with header seq: 21759 +stamp: + secs: 1733935176 + nsecs: 876080989 +frame_id: '' and grasped_object_mask header seq: 505817 +stamp: + secs: 1733935176 + nsecs: 619087219 +frame_id: "camera_color_optical_frame" +[INFO] [1733935177.109427]: received mask_obs with header seq: 21760 +stamp: + secs: 1733935177 + nsecs: 113616704 +frame_id: '' and grasped_object_mask header seq: 505825 +stamp: + secs: 1733935176 + nsecs: 885540724 +frame_id: "camera_color_optical_frame" +[INFO] [1733935177.348459]: received mask_obs with header seq: 21761 +stamp: + secs: 1733935177 + nsecs: 352254390 +frame_id: '' and grasped_object_mask header seq: 505832 +stamp: + secs: 1733935177 + nsecs: 118688583 +frame_id: "camera_color_optical_frame" +[INFO] [1733935177.583805]: received mask_obs with header seq: 21762 +stamp: + secs: 1733935177 + nsecs: 587936162 +frame_id: '' and grasped_object_mask header seq: 505839 +stamp: + secs: 1733935177 + nsecs: 351836443 +frame_id: "camera_color_optical_frame" +[INFO] [1733935177.830876]: received mask_obs with header seq: 21763 +stamp: + secs: 1733935177 + nsecs: 835005044 +frame_id: '' and grasped_object_mask header seq: 505846 +stamp: + secs: 1733935177 + nsecs: 584983587 +frame_id: "camera_color_optical_frame" +[INFO] [1733935178.102598]: received mask_obs with header seq: 21764 +stamp: + secs: 1733935178 + nsecs: 105073451 +frame_id: '' and grasped_object_mask header seq: 505854 +stamp: + secs: 1733935177 + nsecs: 851439238 +frame_id: "camera_color_optical_frame" +[INFO] [1733935178.343552]: received mask_obs with header seq: 21765 +stamp: + secs: 1733935178 + nsecs: 346699953 +frame_id: '' and grasped_object_mask header seq: 505862 +stamp: + secs: 1733935178 + nsecs: 117895126 +frame_id: "camera_color_optical_frame" +[INFO] [1733935178.604505]: received mask_obs with header seq: 21766 +stamp: + secs: 1733935178 + nsecs: 607654571 +frame_id: '' and grasped_object_mask header seq: 505870 +stamp: + secs: 1733935178 + nsecs: 384350777 +frame_id: "camera_color_optical_frame" +[INFO] [1733935178.866867]: received mask_obs with header seq: 21767 +stamp: + secs: 1733935178 + nsecs: 869883537 +frame_id: '' and grasped_object_mask header seq: 505878 +stamp: + secs: 1733935178 + nsecs: 650806904 +frame_id: "camera_color_optical_frame" +[INFO] [1733935179.120445]: received mask_obs with header seq: 21768 +stamp: + secs: 1733935179 + nsecs: 124632835 +frame_id: '' and grasped_object_mask header seq: 505885 +stamp: + secs: 1733935178 + nsecs: 883956671 +frame_id: "camera_color_optical_frame" +[INFO] [1733935179.376016]: received mask_obs with header seq: 21769 +stamp: + secs: 1733935179 + nsecs: 380092144 +frame_id: '' and grasped_object_mask header seq: 505893 +stamp: + secs: 1733935179 + nsecs: 150413036 +frame_id: "camera_color_optical_frame" +[INFO] [1733935179.618412]: received mask_obs with header seq: 21770 +stamp: + secs: 1733935179 + nsecs: 622185468 +frame_id: '' and grasped_object_mask header seq: 505900 +stamp: + secs: 1733935179 + nsecs: 383563280 +frame_id: "camera_color_optical_frame" +[INFO] [1733935179.875143]: received mask_obs with header seq: 21771 +stamp: + secs: 1733935179 + nsecs: 879226446 +frame_id: '' and grasped_object_mask header seq: 505908 +stamp: + secs: 1733935179 + nsecs: 650020123 +frame_id: "camera_color_optical_frame" +[INFO] [1733935180.136465]: received mask_obs with header seq: 21772 +stamp: + secs: 1733935180 + nsecs: 140661954 +frame_id: '' and grasped_object_mask header seq: 505916 +stamp: + secs: 1733935179 + nsecs: 916470289 +frame_id: "camera_color_optical_frame" +[INFO] [1733935180.376741]: received mask_obs with header seq: 21773 +stamp: + secs: 1733935180 + nsecs: 380781412 +frame_id: '' and grasped_object_mask header seq: 505923 +stamp: + secs: 1733935180 + nsecs: 149608612 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 106413964 for nearest: 1733935180489160899 - target: 1733935180382746935 at idx 209 +[INFO] [1733935180.615860]: received mask_obs with header seq: 21774 +stamp: + secs: 1733935180 + nsecs: 620041608 +frame_id: '' and grasped_object_mask header seq: 505930 +stamp: + secs: 1733935180 + nsecs: 382746935 +frame_id: "camera_color_optical_frame" +[INFO] [1733935180.885929]: received mask_obs with header seq: 21775 +stamp: + secs: 1733935180 + nsecs: 890094280 +frame_id: '' and grasped_object_mask header seq: 505938 +stamp: + secs: 1733935180 + nsecs: 649189472 +frame_id: "camera_color_optical_frame" +[INFO] [1733935181.134393]: received mask_obs with header seq: 21776 +stamp: + secs: 1733935181 + nsecs: 138520002 +frame_id: '' and grasped_object_mask header seq: 505946 +stamp: + secs: 1733935180 + nsecs: 915633917 +frame_id: "camera_color_optical_frame" +[INFO] [1733935181.376055]: received mask_obs with header seq: 21777 +stamp: + secs: 1733935181 + nsecs: 379859209 +frame_id: '' and grasped_object_mask header seq: 505953 +stamp: + secs: 1733935181 + nsecs: 148774862 +frame_id: "camera_color_optical_frame" +[INFO] [1733935181.633156]: received mask_obs with header seq: 21778 +stamp: + secs: 1733935181 + nsecs: 637293577 +frame_id: '' and grasped_object_mask header seq: 505960 +stamp: + secs: 1733935181 + nsecs: 381914854 +frame_id: "camera_color_optical_frame" +[INFO] [1733935181.898108]: received mask_obs with header seq: 21779 +stamp: + secs: 1733935181 + nsecs: 900729656 +frame_id: '' and grasped_object_mask header seq: 505968 +stamp: + secs: 1733935181 + nsecs: 648362160 +frame_id: "camera_color_optical_frame" +[INFO] [1733935182.173773]: received mask_obs with header seq: 21780 +stamp: + secs: 1733935182 + nsecs: 177624464 +frame_id: '' and grasped_object_mask header seq: 505976 +stamp: + secs: 1733935181 + nsecs: 915956736 +frame_id: "camera_color_optical_frame" +[INFO] [1733935182.415963]: received mask_obs with header seq: 21781 +stamp: + secs: 1733935182 + nsecs: 420083284 +frame_id: '' and grasped_object_mask header seq: 505984 +stamp: + secs: 1733935182 + nsecs: 185570955 +frame_id: "camera_color_optical_frame" +[INFO] [1733935182.679034]: received mask_obs with header seq: 21782 +stamp: + secs: 1733935182 + nsecs: 682882308 +frame_id: '' and grasped_object_mask header seq: 505992 +stamp: + secs: 1733935182 + nsecs: 455341339 +frame_id: "camera_color_optical_frame" +[INFO] [1733935182.938019]: received mask_obs with header seq: 21783 +stamp: + secs: 1733935182 + nsecs: 942092657 +frame_id: '' and grasped_object_mask header seq: 506000 +stamp: + secs: 1733935182 + nsecs: 725268364 +frame_id: "camera_color_optical_frame" +[INFO] [1733935183.192091]: received mask_obs with header seq: 21784 +stamp: + secs: 1733935183 + nsecs: 196298599 +frame_id: '' and grasped_object_mask header seq: 506007 +stamp: + secs: 1733935182 + nsecs: 959615946 +frame_id: "camera_color_optical_frame" +[INFO] [1733935183.453840]: received mask_obs with header seq: 21785 +stamp: + secs: 1733935183 + nsecs: 457088470 +frame_id: '' and grasped_object_mask header seq: 506015 +stamp: + secs: 1733935183 + nsecs: 225772381 +frame_id: "camera_color_optical_frame" +[INFO] [1733935183.707251]: received mask_obs with header seq: 21786 +stamp: + secs: 1733935183 + nsecs: 710543632 +frame_id: '' and grasped_object_mask header seq: 506023 +stamp: + secs: 1733935183 + nsecs: 491910219 +frame_id: "camera_color_optical_frame" +[INFO] [1733935183.963027]: received mask_obs with header seq: 21787 +stamp: + secs: 1733935183 + nsecs: 966127872 +frame_id: '' and grasped_object_mask header seq: 506031 +stamp: + secs: 1733935183 + nsecs: 758027077 +frame_id: "camera_color_optical_frame" +[INFO] [1733935184.225902]: received mask_obs with header seq: 21788 +stamp: + secs: 1733935184 + nsecs: 229969501 +frame_id: '' and grasped_object_mask header seq: 506038 +stamp: + secs: 1733935183 + nsecs: 990893841 +frame_id: "camera_color_optical_frame" +[INFO] [1733935184.491910]: received mask_obs with header seq: 21789 +stamp: + secs: 1733935184 + nsecs: 496121644 +frame_id: '' and grasped_object_mask header seq: 506046 +stamp: + secs: 1733935184 + nsecs: 257020473 +frame_id: "camera_color_optical_frame" +[INFO] [1733935184.747348]: received mask_obs with header seq: 21790 +stamp: + secs: 1733935184 + nsecs: 751097917 +frame_id: '' and grasped_object_mask header seq: 506054 +stamp: + secs: 1733935184 + nsecs: 523125887 +frame_id: "camera_color_optical_frame" +[INFO] [1733935185.004130]: received mask_obs with header seq: 21791 +stamp: + secs: 1733935185 + nsecs: 7872819 +frame_id: '' and grasped_object_mask header seq: 506062 +stamp: + secs: 1733935184 + nsecs: 789209843 +frame_id: "camera_color_optical_frame" +[INFO] [1733935185.243408]: received mask_obs with header seq: 21792 +stamp: + secs: 1733935185 + nsecs: 247568368 +frame_id: '' and grasped_object_mask header seq: 506069 +stamp: + secs: 1733935185 + nsecs: 22075653 +frame_id: "camera_color_optical_frame" +[INFO] [1733935185.500817]: received mask_obs with header seq: 21793 +stamp: + secs: 1733935185 + nsecs: 504896879 +frame_id: '' and grasped_object_mask header seq: 506077 +stamp: + secs: 1733935185 + nsecs: 288198471 +frame_id: "camera_color_optical_frame" +[INFO] [1733935185.770385]: received mask_obs with header seq: 21794 +stamp: + secs: 1733935185 + nsecs: 774521112 +frame_id: '' and grasped_object_mask header seq: 506085 +stamp: + secs: 1733935185 + nsecs: 554299593 +frame_id: "camera_color_optical_frame" +[INFO] [1733935186.028264]: received mask_obs with header seq: 21795 +stamp: + secs: 1733935186 + nsecs: 32191276 +frame_id: '' and grasped_object_mask header seq: 506093 +stamp: + secs: 1733935185 + nsecs: 820380449 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -130713275 for nearest: 1733935185922545382 - target: 1733935186053258657 at idx 249 +[INFO] [1733935186.284887]: received mask_obs with header seq: 21796 +stamp: + secs: 1733935186 + nsecs: 289029359 +frame_id: '' and grasped_object_mask header seq: 506100 +stamp: + secs: 1733935186 + nsecs: 53258657 +frame_id: "camera_color_optical_frame" +[INFO] [1733935186.562553]: received mask_obs with header seq: 21797 +stamp: + secs: 1733935186 + nsecs: 566272497 +frame_id: '' and grasped_object_mask header seq: 506108 +stamp: + secs: 1733935186 + nsecs: 319385767 +frame_id: "camera_color_optical_frame" +[INFO] [1733935186.832018]: received mask_obs with header seq: 21798 +stamp: + secs: 1733935186 + nsecs: 835684776 +frame_id: '' and grasped_object_mask header seq: 506116 +stamp: + secs: 1733935186 + nsecs: 585492849 +frame_id: "camera_color_optical_frame" +[INFO] [1733935187.097888]: received mask_obs with header seq: 21799 +stamp: + secs: 1733935187 + nsecs: 101968526 +frame_id: '' and grasped_object_mask header seq: 506124 +stamp: + secs: 1733935186 + nsecs: 851588488 +frame_id: "camera_color_optical_frame" +[INFO] [1733935187.341339]: received mask_obs with header seq: 21800 +stamp: + secs: 1733935187 + nsecs: 345476865 +frame_id: '' and grasped_object_mask header seq: 506132 +stamp: + secs: 1733935187 + nsecs: 117734909 +frame_id: "camera_color_optical_frame" +[INFO] [1733935187.593922]: received mask_obs with header seq: 21801 +stamp: + secs: 1733935187 + nsecs: 597943067 +frame_id: '' and grasped_object_mask header seq: 506140 +stamp: + secs: 1733935187 + nsecs: 383860350 +frame_id: "camera_color_optical_frame" +[INFO] [1733935187.842028]: received mask_obs with header seq: 21802 +stamp: + secs: 1733935187 + nsecs: 846242904 +frame_id: '' and grasped_object_mask header seq: 506147 +stamp: + secs: 1733935187 + nsecs: 616703510 +frame_id: "camera_color_optical_frame" +[INFO] [1733935188.080121]: received mask_obs with header seq: 21803 +stamp: + secs: 1733935188 + nsecs: 84244728 +frame_id: '' and grasped_object_mask header seq: 506154 +stamp: + secs: 1733935187 + nsecs: 849540234 +frame_id: "camera_color_optical_frame" +[INFO] [1733935188.325378]: received mask_obs with header seq: 21804 +stamp: + secs: 1733935188 + nsecs: 328027248 +frame_id: '' and grasped_object_mask header seq: 506162 +stamp: + secs: 1733935188 + nsecs: 115684032 +frame_id: "camera_color_optical_frame" +[INFO] [1733935188.575708]: received mask_obs with header seq: 21805 +stamp: + secs: 1733935188 + nsecs: 579791307 +frame_id: '' and grasped_object_mask header seq: 506169 +stamp: + secs: 1733935188 + nsecs: 348543406 +frame_id: "camera_color_optical_frame" +[INFO] [1733935188.817694]: received mask_obs with header seq: 21806 +stamp: + secs: 1733935188 + nsecs: 821551561 +frame_id: '' and grasped_object_mask header seq: 506176 +stamp: + secs: 1733935188 + nsecs: 581385851 +frame_id: "camera_color_optical_frame" +[INFO] [1733935189.080399]: received mask_obs with header seq: 21807 +stamp: + secs: 1733935189 + nsecs: 84371805 +frame_id: '' and grasped_object_mask header seq: 506184 +stamp: + secs: 1733935188 + nsecs: 847512960 +frame_id: "camera_color_optical_frame" +[INFO] [1733935189.334785]: received mask_obs with header seq: 21808 +stamp: + secs: 1733935189 + nsecs: 338639974 +frame_id: '' and grasped_object_mask header seq: 506192 +stamp: + secs: 1733935189 + nsecs: 113715887 +frame_id: "camera_color_optical_frame" +[INFO] [1733935189.580146]: received mask_obs with header seq: 21809 +stamp: + secs: 1733935189 + nsecs: 584160327 +frame_id: '' and grasped_object_mask header seq: 506199 +stamp: + secs: 1733935189 + nsecs: 346626997 +frame_id: "camera_color_optical_frame" +[INFO] [1733935189.839795]: received mask_obs with header seq: 21810 +stamp: + secs: 1733935189 + nsecs: 843528747 +frame_id: '' and grasped_object_mask header seq: 506207 +stamp: + secs: 1733935189 + nsecs: 612795830 +frame_id: "camera_color_optical_frame" +[INFO] [1733935190.094327]: received mask_obs with header seq: 21811 +stamp: + secs: 1733935190 + nsecs: 97571611 +frame_id: '' and grasped_object_mask header seq: 506214 +stamp: + secs: 1733935189 + nsecs: 845677376 +frame_id: "camera_color_optical_frame" +[INFO] [1733935190.335316]: received mask_obs with header seq: 21812 +stamp: + secs: 1733935190 + nsecs: 338675260 +frame_id: '' and grasped_object_mask header seq: 506222 +stamp: + secs: 1733935190 + nsecs: 111813307 +frame_id: "camera_color_optical_frame" +[INFO] [1733935190.591647]: received mask_obs with header seq: 21813 +stamp: + secs: 1733935190 + nsecs: 595306158 +frame_id: '' and grasped_object_mask header seq: 506230 +stamp: + secs: 1733935190 + nsecs: 377928257 +frame_id: "camera_color_optical_frame" +[INFO] [1733935190.831318]: received mask_obs with header seq: 21814 +stamp: + secs: 1733935190 + nsecs: 835167407 +frame_id: '' and grasped_object_mask header seq: 506237 +stamp: + secs: 1733935190 + nsecs: 610759735 +frame_id: "camera_color_optical_frame" +[INFO] [1733935191.071929]: received mask_obs with header seq: 21815 +stamp: + secs: 1733935191 + nsecs: 76062440 +frame_id: '' and grasped_object_mask header seq: 506244 +stamp: + secs: 1733935190 + nsecs: 843576908 +frame_id: "camera_color_optical_frame" +[INFO] [1733935191.307899]: received mask_obs with header seq: 21816 +stamp: + secs: 1733935191 + nsecs: 312036037 +frame_id: '' and grasped_object_mask header seq: 506251 +stamp: + secs: 1733935191 + nsecs: 76452255 +frame_id: "camera_color_optical_frame" +[INFO] [1733935191.556067]: received mask_obs with header seq: 21817 +stamp: + secs: 1733935191 + nsecs: 558880090 +frame_id: '' and grasped_object_mask header seq: 506259 +stamp: + secs: 1733935191 + nsecs: 342575312 +frame_id: "camera_color_optical_frame" +[INFO] [1733935191.804610]: received mask_obs with header seq: 21818 +stamp: + secs: 1733935191 + nsecs: 808757066 +frame_id: '' and grasped_object_mask header seq: 506266 +stamp: + secs: 1733935191 + nsecs: 575416088 +frame_id: "camera_color_optical_frame" +[INFO] [1733935192.050663]: received mask_obs with header seq: 21819 +stamp: + secs: 1733935192 + nsecs: 54816722 +frame_id: '' and grasped_object_mask header seq: 506273 +stamp: + secs: 1733935191 + nsecs: 808239698 +frame_id: "camera_color_optical_frame" +[INFO] [1733935192.304916]: received mask_obs with header seq: 21820 +stamp: + secs: 1733935192 + nsecs: 309101343 +frame_id: '' and grasped_object_mask header seq: 506281 +stamp: + secs: 1733935192 + nsecs: 74379444 +frame_id: "camera_color_optical_frame" +[INFO] [1733935192.548904]: received mask_obs with header seq: 21821 +stamp: + secs: 1733935192 + nsecs: 553007364 +frame_id: '' and grasped_object_mask header seq: 506288 +stamp: + secs: 1733935192 + nsecs: 307241201 +frame_id: "camera_color_optical_frame" +[INFO] [1733935192.804166]: received mask_obs with header seq: 21822 +stamp: + secs: 1733935192 + nsecs: 808112382 +frame_id: '' and grasped_object_mask header seq: 506296 +stamp: + secs: 1733935192 + nsecs: 573350191 +frame_id: "camera_color_optical_frame" +[INFO] [1733935193.057770]: received mask_obs with header seq: 21823 +stamp: + secs: 1733935193 + nsecs: 61841011 +frame_id: '' and grasped_object_mask header seq: 506303 +stamp: + secs: 1733935192 + nsecs: 806178808 +frame_id: "camera_color_optical_frame" +[INFO] [1733935193.303799]: received mask_obs with header seq: 21824 +stamp: + secs: 1733935193 + nsecs: 307886600 +frame_id: '' and grasped_object_mask header seq: 506311 +stamp: + secs: 1733935193 + nsecs: 72315454 +frame_id: "camera_color_optical_frame" +[INFO] [1733935193.548993]: received mask_obs with header seq: 21825 +stamp: + secs: 1733935193 + nsecs: 553128719 +frame_id: '' and grasped_object_mask header seq: 506318 +stamp: + secs: 1733935193 + nsecs: 305175781 +frame_id: "camera_color_optical_frame" +[INFO] [1733935193.802742]: received mask_obs with header seq: 21826 +stamp: + secs: 1733935193 + nsecs: 806644678 +frame_id: '' and grasped_object_mask header seq: 506326 +stamp: + secs: 1733935193 + nsecs: 571281433 +frame_id: "camera_color_optical_frame" +[INFO] [1733935194.059201]: received mask_obs with header seq: 21827 +stamp: + secs: 1733935194 + nsecs: 63362121 +frame_id: '' and grasped_object_mask header seq: 506334 +stamp: + secs: 1733935193 + nsecs: 837368011 +frame_id: "camera_color_optical_frame" +[INFO] [1733935194.300064]: received mask_obs with header seq: 21828 +stamp: + secs: 1733935194 + nsecs: 304279565 +frame_id: '' and grasped_object_mask header seq: 506341 +stamp: + secs: 1733935194 + nsecs: 70242405 +frame_id: "camera_color_optical_frame" +[INFO] [1733935194.557271]: received mask_obs with header seq: 21829 +stamp: + secs: 1733935194 + nsecs: 561432123 +frame_id: '' and grasped_object_mask header seq: 506349 +stamp: + secs: 1733935194 + nsecs: 336365461 +frame_id: "camera_color_optical_frame" +[INFO] [1733935194.820485]: received mask_obs with header seq: 21830 +stamp: + secs: 1733935194 + nsecs: 823925495 +frame_id: '' and grasped_object_mask header seq: 506357 +stamp: + secs: 1733935194 + nsecs: 602468252 +frame_id: "camera_color_optical_frame" +[INFO] [1733935195.075405]: received mask_obs with header seq: 21831 +stamp: + secs: 1733935195 + nsecs: 79378128 +frame_id: '' and grasped_object_mask header seq: 506364 +stamp: + secs: 1733935194 + nsecs: 835289955 +frame_id: "camera_color_optical_frame" +[INFO] [1733935195.334019]: received mask_obs with header seq: 21832 +stamp: + secs: 1733935195 + nsecs: 338106870 +frame_id: '' and grasped_object_mask header seq: 506372 +stamp: + secs: 1733935195 + nsecs: 101439238 +frame_id: "camera_color_optical_frame" +[INFO] [1733935195.599771]: received mask_obs with header seq: 21833 +stamp: + secs: 1733935195 + nsecs: 603859901 +frame_id: '' and grasped_object_mask header seq: 506380 +stamp: + secs: 1733935195 + nsecs: 367566824 +frame_id: "camera_color_optical_frame" +[INFO] [1733935195.856320]: received mask_obs with header seq: 21834 +stamp: + secs: 1733935195 + nsecs: 860515832 +frame_id: '' and grasped_object_mask header seq: 506387 +stamp: + secs: 1733935195 + nsecs: 600410938 +frame_id: "camera_color_optical_frame" +[INFO] [1733935196.119728]: received mask_obs with header seq: 21835 +stamp: + secs: 1733935196 + nsecs: 123861789 +frame_id: '' and grasped_object_mask header seq: 506396 +stamp: + secs: 1733935195 + nsecs: 899779081 +frame_id: "camera_color_optical_frame" +[INFO] [1733935196.376211]: received mask_obs with header seq: 21836 +stamp: + secs: 1733935196 + nsecs: 380432844 +frame_id: '' and grasped_object_mask header seq: 506403 +stamp: + secs: 1733935196 + nsecs: 132656336 +frame_id: "camera_color_optical_frame" +[INFO] [1733935196.632229]: received mask_obs with header seq: 21837 +stamp: + secs: 1733935196 + nsecs: 636355400 +frame_id: '' and grasped_object_mask header seq: 506411 +stamp: + secs: 1733935196 + nsecs: 398781061 +frame_id: "camera_color_optical_frame" +[INFO] [1733935196.875585]: received mask_obs with header seq: 21838 +stamp: + secs: 1733935196 + nsecs: 879446268 +frame_id: '' and grasped_object_mask header seq: 506418 +stamp: + secs: 1733935196 + nsecs: 631623507 +frame_id: "camera_color_optical_frame" +[INFO] [1733935197.152243]: received mask_obs with header seq: 21839 +stamp: + secs: 1733935197 + nsecs: 154702186 +frame_id: '' and grasped_object_mask header seq: 506426 +stamp: + secs: 1733935196 + nsecs: 897722960 +frame_id: "camera_color_optical_frame" +[INFO] [1733935197.409819]: received mask_obs with header seq: 21840 +stamp: + secs: 1733935197 + nsecs: 413322448 +frame_id: '' and grasped_object_mask header seq: 506434 +stamp: + secs: 1733935197 + nsecs: 163841963 +frame_id: "camera_color_optical_frame" +[INFO] [1733935197.666613]: received mask_obs with header seq: 21841 +stamp: + secs: 1733935197 + nsecs: 670835971 +frame_id: '' and grasped_object_mask header seq: 506442 +stamp: + secs: 1733935197 + nsecs: 429939032 +frame_id: "camera_color_optical_frame" +[INFO] [1733935197.908238]: received mask_obs with header seq: 21842 +stamp: + secs: 1733935197 + nsecs: 912391662 +frame_id: '' and grasped_object_mask header seq: 506449 +stamp: + secs: 1733935197 + nsecs: 662755728 +frame_id: "camera_color_optical_frame" +[INFO] [1733935198.152999]: received mask_obs with header seq: 21843 +stamp: + secs: 1733935198 + nsecs: 155059337 +frame_id: '' and grasped_object_mask header seq: 506457 +stamp: + secs: 1733935197 + nsecs: 929482698 +frame_id: "camera_color_optical_frame" +[INFO] [1733935198.401268]: received mask_obs with header seq: 21844 +stamp: + secs: 1733935198 + nsecs: 405369520 +frame_id: '' and grasped_object_mask header seq: 506464 +stamp: + secs: 1733935198 + nsecs: 163929462 +frame_id: "camera_color_optical_frame" +[INFO] [1733935198.647663]: received mask_obs with header seq: 21845 +stamp: + secs: 1733935198 + nsecs: 651530504 +frame_id: '' and grasped_object_mask header seq: 506471 +stamp: + secs: 1733935198 + nsecs: 398495913 +frame_id: "camera_color_optical_frame" +[INFO] [1733935198.900446]: received mask_obs with header seq: 21846 +stamp: + secs: 1733935198 + nsecs: 904503822 +frame_id: '' and grasped_object_mask header seq: 506479 +stamp: + secs: 1733935198 + nsecs: 666717768 +frame_id: "camera_color_optical_frame" +[INFO] [1733935199.148461]: received mask_obs with header seq: 21847 +stamp: + secs: 1733935199 + nsecs: 151772022 +frame_id: '' and grasped_object_mask header seq: 506487 +stamp: + secs: 1733935198 + nsecs: 934359550 +frame_id: "camera_color_optical_frame" +[INFO] [1733935199.400551]: received mask_obs with header seq: 21848 +stamp: + secs: 1733935199 + nsecs: 404633283 +frame_id: '' and grasped_object_mask header seq: 506494 +stamp: + secs: 1733935199 + nsecs: 167516232 +frame_id: "camera_color_optical_frame" +[INFO] [1733935199.649087]: received mask_obs with header seq: 21849 +stamp: + secs: 1733935199 + nsecs: 651871919 +frame_id: '' and grasped_object_mask header seq: 506501 +stamp: + secs: 1733935199 + nsecs: 400673628 +frame_id: "camera_color_optical_frame" +[INFO] [1733935199.902720]: received mask_obs with header seq: 21850 +stamp: + secs: 1733935199 + nsecs: 906804561 +frame_id: '' and grasped_object_mask header seq: 506509 +stamp: + secs: 1733935199 + nsecs: 667139530 +frame_id: "camera_color_optical_frame" +[INFO] [1733935200.147084]: received mask_obs with header seq: 21851 +stamp: + secs: 1733935200 + nsecs: 151091098 +frame_id: '' and grasped_object_mask header seq: 506516 +stamp: + secs: 1733935199 + nsecs: 900291920 +frame_id: "camera_color_optical_frame" +[INFO] [1733935200.398317]: received mask_obs with header seq: 21852 +stamp: + secs: 1733935200 + nsecs: 402060747 +frame_id: '' and grasped_object_mask header seq: 506524 +stamp: + secs: 1733935200 + nsecs: 166736603 +frame_id: "camera_color_optical_frame" +[INFO] [1733935200.634926]: received mask_obs with header seq: 21853 +stamp: + secs: 1733935200 + nsecs: 639084815 +frame_id: '' and grasped_object_mask header seq: 506531 +stamp: + secs: 1733935200 + nsecs: 399873734 +frame_id: "camera_color_optical_frame" +[INFO] [1733935200.896180]: received mask_obs with header seq: 21854 +stamp: + secs: 1733935200 + nsecs: 900157451 +frame_id: '' and grasped_object_mask header seq: 506539 +stamp: + secs: 1733935200 + nsecs: 666317463 +frame_id: "camera_color_optical_frame" +[INFO] [1733935201.152627]: received mask_obs with header seq: 21855 +stamp: + secs: 1733935201 + nsecs: 156271219 +frame_id: '' and grasped_object_mask header seq: 506547 +stamp: + secs: 1733935200 + nsecs: 932763338 +frame_id: "camera_color_optical_frame" +[INFO] [1733935201.390256]: received mask_obs with header seq: 21856 +stamp: + secs: 1733935201 + nsecs: 394433259 +frame_id: '' and grasped_object_mask header seq: 506554 +stamp: + secs: 1733935201 + nsecs: 165908575 +frame_id: "camera_color_optical_frame" +[INFO] [1733935201.490599]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1733935201.631024]: received mask_obs with header seq: 21857 +stamp: + secs: 1733935201 + nsecs: 633050203 +frame_id: '' and grasped_object_mask header seq: 506561 +stamp: + secs: 1733935201 + nsecs: 399055243 +frame_id: "camera_color_optical_frame" +[INFO] [1733935202.252853]: Stored episode 3. +[INFO] [1733935202.258044]: Saving video... +[INFO] [1733935202.471403]: Video saved! +Starting episode 3 +[INFO] [1733935202.502581]: resetting environment +[INFO] [1733935202.527956]: cleared current plan +[INFO] [1733935202.528250]: moving to home +[INFO] [1733935205.331979]: reached home +[INFO] [1733935205.333119]: reset action history +[INFO] [1733935208.055660]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1733935209.531352]: resetting environment +[INFO] [1733935209.554355]: cleared current plan +[INFO] [1733935209.554556]: moving to home +[INFO] [1733935210.556574]: reached home +[INFO] [1733935210.561862]: reset action history +[INFO] [1733935213.309121]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/episode_rosbags/episode_3_2024-12-11-11-40-13.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-743294.log +started roslaunch server http://158.130.50.37:36721/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [744059] +process[print_text-11]: started with pid [744060] +process[rosbag_record-12]: started with pid [744061] +started bagging! +[INFO] [1733935215.089853]: received mask_obs with header seq: 21858 +stamp: + secs: 1733935215 + nsecs: 93726158 +frame_id: '' and grasped_object_mask header seq: 506966 +stamp: + secs: 1733935214 + nsecs: 895430088 +frame_id: "camera_color_optical_frame" +[INFO] [1733935215.323558]: received mask_obs with header seq: 21859 +stamp: + secs: 1733935215 + nsecs: 327698707 +frame_id: '' and grasped_object_mask header seq: 506973 +stamp: + secs: 1733935215 + nsecs: 128320217 +frame_id: "camera_color_optical_frame" +[INFO] [1733935215.555022]: received mask_obs with header seq: 21860 +stamp: + secs: 1733935215 + nsecs: 559146404 +frame_id: '' and grasped_object_mask header seq: 506979 +stamp: + secs: 1733935215 + nsecs: 327931404 +frame_id: "camera_color_optical_frame" +[INFO] [1733935215.792666]: received mask_obs with header seq: 21861 +stamp: + secs: 1733935215 + nsecs: 796848058 +frame_id: '' and grasped_object_mask header seq: 506987 +stamp: + secs: 1733935215 + nsecs: 594063044 +frame_id: "camera_color_optical_frame" +[INFO] [1733935216.033469]: received mask_obs with header seq: 21862 +stamp: + secs: 1733935216 + nsecs: 37598609 +frame_id: '' and grasped_object_mask header seq: 506994 +stamp: + secs: 1733935215 + nsecs: 826910734 +frame_id: "camera_color_optical_frame" +[INFO] [1733935216.275175]: received mask_obs with header seq: 21863 +stamp: + secs: 1733935216 + nsecs: 278962850 +frame_id: '' and grasped_object_mask header seq: 507001 +stamp: + secs: 1733935216 + nsecs: 59787035 +frame_id: "camera_color_optical_frame" +[INFO] [1733935216.513852]: received mask_obs with header seq: 21864 +stamp: + secs: 1733935216 + nsecs: 517813920 +frame_id: '' and grasped_object_mask header seq: 507008 +stamp: + secs: 1733935216 + nsecs: 292669058 +frame_id: "camera_color_optical_frame" +[INFO] [1733935216.757042]: received mask_obs with header seq: 21865 +stamp: + secs: 1733935216 + nsecs: 760797262 +frame_id: '' and grasped_object_mask header seq: 507015 +stamp: + secs: 1733935216 + nsecs: 525534630 +frame_id: "camera_color_optical_frame" +[INFO] [1733935217.014708]: received mask_obs with header seq: 21866 +stamp: + secs: 1733935217 + nsecs: 18751144 +frame_id: '' and grasped_object_mask header seq: 507023 +stamp: + secs: 1733935216 + nsecs: 791649580 +frame_id: "camera_color_optical_frame" +[INFO] [1733935217.268112]: received mask_obs with header seq: 21867 +stamp: + secs: 1733935217 + nsecs: 272260427 +frame_id: '' and grasped_object_mask header seq: 507030 +stamp: + secs: 1733935217 + nsecs: 24523258 +frame_id: "camera_color_optical_frame" +[INFO] [1733935217.521808]: received mask_obs with header seq: 21868 +stamp: + secs: 1733935217 + nsecs: 525960922 +frame_id: '' and grasped_object_mask header seq: 507038 +stamp: + secs: 1733935217 + nsecs: 290689707 +frame_id: "camera_color_optical_frame" +[INFO] [1733935217.763254]: received mask_obs with header seq: 21869 +stamp: + secs: 1733935217 + nsecs: 767248153 +frame_id: '' and grasped_object_mask header seq: 507045 +stamp: + secs: 1733935217 + nsecs: 523568392 +frame_id: "camera_color_optical_frame" +[INFO] [1733935218.019636]: received mask_obs with header seq: 21870 +stamp: + secs: 1733935218 + nsecs: 23809909 +frame_id: '' and grasped_object_mask header seq: 507053 +stamp: + secs: 1733935217 + nsecs: 789697886 +frame_id: "camera_color_optical_frame" +[INFO] [1733935218.258958]: received mask_obs with header seq: 21871 +stamp: + secs: 1733935218 + nsecs: 263072729 +frame_id: '' and grasped_object_mask header seq: 507060 +stamp: + secs: 1733935218 + nsecs: 22571325 +frame_id: "camera_color_optical_frame" +[INFO] [1733935218.519439]: received mask_obs with header seq: 21872 +stamp: + secs: 1733935218 + nsecs: 523592472 +frame_id: '' and grasped_object_mask header seq: 507068 +stamp: + secs: 1733935218 + nsecs: 288718462 +frame_id: "camera_color_optical_frame" +[INFO] [1733935218.780313]: received mask_obs with header seq: 21873 +stamp: + secs: 1733935218 + nsecs: 784482717 +frame_id: '' and grasped_object_mask header seq: 507076 +stamp: + secs: 1733935218 + nsecs: 554846525 +frame_id: "camera_color_optical_frame" +[INFO] [1733935219.040540]: received mask_obs with header seq: 21874 +stamp: + secs: 1733935219 + nsecs: 43464660 +frame_id: '' and grasped_object_mask header seq: 507084 +stamp: + secs: 1733935218 + nsecs: 820952654 +frame_id: "camera_color_optical_frame" +[INFO] [1733935219.292386]: received mask_obs with header seq: 21875 +stamp: + secs: 1733935219 + nsecs: 296561002 +frame_id: '' and grasped_object_mask header seq: 507091 +stamp: + secs: 1733935219 + nsecs: 53821802 +frame_id: "camera_color_optical_frame" +[INFO] [1733935219.533554]: received mask_obs with header seq: 21876 +stamp: + secs: 1733935219 + nsecs: 537701845 +frame_id: '' and grasped_object_mask header seq: 507098 +stamp: + secs: 1733935219 + nsecs: 286697865 +frame_id: "camera_color_optical_frame" +[INFO] [1733935219.788468]: received mask_obs with header seq: 21877 +stamp: + secs: 1733935219 + nsecs: 792135953 +frame_id: '' and grasped_object_mask header seq: 507106 +stamp: + secs: 1733935219 + nsecs: 552822113 +frame_id: "camera_color_optical_frame" +[INFO] [1733935220.045468]: received mask_obs with header seq: 21878 +stamp: + secs: 1733935220 + nsecs: 49114942 +frame_id: '' and grasped_object_mask header seq: 507114 +stamp: + secs: 1733935219 + nsecs: 818927288 +frame_id: "camera_color_optical_frame" +[INFO] [1733935220.286888]: received mask_obs with header seq: 21879 +stamp: + secs: 1733935220 + nsecs: 290948629 +frame_id: '' and grasped_object_mask header seq: 507121 +stamp: + secs: 1733935220 + nsecs: 51789999 +frame_id: "camera_color_optical_frame" +[INFO] [1733935220.541257]: received mask_obs with header seq: 21880 +stamp: + secs: 1733935220 + nsecs: 544214248 +frame_id: '' and grasped_object_mask header seq: 507129 +stamp: + secs: 1733935220 + nsecs: 317922831 +frame_id: "camera_color_optical_frame" +[INFO] [1733935220.779520]: received mask_obs with header seq: 21881 +stamp: + secs: 1733935220 + nsecs: 783396720 +frame_id: '' and grasped_object_mask header seq: 507136 +stamp: + secs: 1733935220 + nsecs: 550771952 +frame_id: "camera_color_optical_frame" +[INFO] [1733935221.042099]: received mask_obs with header seq: 21882 +stamp: + secs: 1733935221 + nsecs: 45156717 +frame_id: '' and grasped_object_mask header seq: 507144 +stamp: + secs: 1733935220 + nsecs: 816865206 +frame_id: "camera_color_optical_frame" +[INFO] [1733935221.300535]: received mask_obs with header seq: 21883 +stamp: + secs: 1733935221 + nsecs: 304654121 +frame_id: '' and grasped_object_mask header seq: 507151 +stamp: + secs: 1733935221 + nsecs: 49725294 +frame_id: "camera_color_optical_frame" +[INFO] [1733935221.557344]: received mask_obs with header seq: 21884 +stamp: + secs: 1733935221 + nsecs: 561454057 +frame_id: '' and grasped_object_mask header seq: 507159 +stamp: + secs: 1733935221 + nsecs: 315857410 +frame_id: "camera_color_optical_frame" +[INFO] [1733935221.797619]: received mask_obs with header seq: 21885 +stamp: + secs: 1733935221 + nsecs: 801739215 +frame_id: '' and grasped_object_mask header seq: 507166 +stamp: + secs: 1733935221 + nsecs: 548706293 +frame_id: "camera_color_optical_frame" +[INFO] [1733935222.073749]: received mask_obs with header seq: 21886 +stamp: + secs: 1733935222 + nsecs: 77609300 +frame_id: '' and grasped_object_mask header seq: 507174 +stamp: + secs: 1733935221 + nsecs: 814798832 +frame_id: "camera_color_optical_frame" +[INFO] [1733935222.310620]: received mask_obs with header seq: 21887 +stamp: + secs: 1733935222 + nsecs: 314741134 +frame_id: '' and grasped_object_mask header seq: 507182 +stamp: + secs: 1733935222 + nsecs: 80936432 +frame_id: "camera_color_optical_frame" +[INFO] [1733935222.545362]: received mask_obs with header seq: 21888 +stamp: + secs: 1733935222 + nsecs: 549525976 +frame_id: '' and grasped_object_mask header seq: 507189 +stamp: + secs: 1733935222 + nsecs: 313815594 +frame_id: "camera_color_optical_frame" +[INFO] [1733935222.787866]: received mask_obs with header seq: 21889 +stamp: + secs: 1733935222 + nsecs: 791800022 +frame_id: '' and grasped_object_mask header seq: 507196 +stamp: + secs: 1733935222 + nsecs: 546679258 +frame_id: "camera_color_optical_frame" +[INFO] [1733935223.053449]: received mask_obs with header seq: 21890 +stamp: + secs: 1733935223 + nsecs: 56632041 +frame_id: '' and grasped_object_mask header seq: 507204 +stamp: + secs: 1733935222 + nsecs: 812789679 +frame_id: "camera_color_optical_frame" +[INFO] [1733935223.297202]: received mask_obs with header seq: 21891 +stamp: + secs: 1733935223 + nsecs: 301276683 +frame_id: '' and grasped_object_mask header seq: 507211 +stamp: + secs: 1733935223 + nsecs: 45667887 +frame_id: "camera_color_optical_frame" +[INFO] [1733935223.539371]: received mask_obs with header seq: 21892 +stamp: + secs: 1733935223 + nsecs: 543291807 +frame_id: '' and grasped_object_mask header seq: 507219 +stamp: + secs: 1733935223 + nsecs: 311823368 +frame_id: "camera_color_optical_frame" +[INFO] [1733935223.796039]: received mask_obs with header seq: 21893 +stamp: + secs: 1733935223 + nsecs: 800148963 +frame_id: '' and grasped_object_mask header seq: 507226 +stamp: + secs: 1733935223 + nsecs: 544693470 +frame_id: "camera_color_optical_frame" +[INFO] [1733935224.038817]: received mask_obs with header seq: 21894 +stamp: + secs: 1733935224 + nsecs: 42830228 +frame_id: '' and grasped_object_mask header seq: 507234 +stamp: + secs: 1733935223 + nsecs: 810811281 +frame_id: "camera_color_optical_frame" +[INFO] [1733935224.295921]: received mask_obs with header seq: 21895 +stamp: + secs: 1733935224 + nsecs: 300125598 +frame_id: '' and grasped_object_mask header seq: 507241 +stamp: + secs: 1733935224 + nsecs: 44837475 +frame_id: "camera_color_optical_frame" +[INFO] [1733935224.553663]: received mask_obs with header seq: 21896 +stamp: + secs: 1733935224 + nsecs: 557424545 +frame_id: '' and grasped_object_mask header seq: 507249 +stamp: + secs: 1733935224 + nsecs: 314125776 +frame_id: "camera_color_optical_frame" +[INFO] [1733935224.792102]: received mask_obs with header seq: 21897 +stamp: + secs: 1733935224 + nsecs: 796225786 +frame_id: '' and grasped_object_mask header seq: 507256 +stamp: + secs: 1733935224 + nsecs: 549862146 +frame_id: "camera_color_optical_frame" +[INFO] [1733935225.042664]: received mask_obs with header seq: 21898 +stamp: + secs: 1733935225 + nsecs: 46849966 +frame_id: '' and grasped_object_mask header seq: 507264 +stamp: + secs: 1733935224 + nsecs: 819402456 +frame_id: "camera_color_optical_frame" +[INFO] [1733935225.293465]: received mask_obs with header seq: 21899 +stamp: + secs: 1733935225 + nsecs: 296850204 +frame_id: '' and grasped_object_mask header seq: 507271 +stamp: + secs: 1733935225 + nsecs: 53925753 +frame_id: "camera_color_optical_frame" +[INFO] [1733935225.552946]: received mask_obs with header seq: 21900 +stamp: + secs: 1733935225 + nsecs: 556694030 +frame_id: '' and grasped_object_mask header seq: 507279 +stamp: + secs: 1733935225 + nsecs: 319777012 +frame_id: "camera_color_optical_frame" +[INFO] [1733935225.808584]: received mask_obs with header seq: 21901 +stamp: + secs: 1733935225 + nsecs: 812628746 +frame_id: '' and grasped_object_mask header seq: 507287 +stamp: + secs: 1733935225 + nsecs: 585587025 +frame_id: "camera_color_optical_frame" +[INFO] [1733935226.069200]: received mask_obs with header seq: 21902 +stamp: + secs: 1733935226 + nsecs: 73320627 +frame_id: '' and grasped_object_mask header seq: 507295 +stamp: + secs: 1733935225 + nsecs: 851356268 +frame_id: "camera_color_optical_frame" +[INFO] [1733935226.322474]: received mask_obs with header seq: 21903 +stamp: + secs: 1733935226 + nsecs: 326528072 +frame_id: '' and grasped_object_mask header seq: 507302 +stamp: + secs: 1733935226 + nsecs: 84779739 +frame_id: "camera_color_optical_frame" +[INFO] [1733935226.575333]: received mask_obs with header seq: 21904 +stamp: + secs: 1733935226 + nsecs: 578642845 +frame_id: '' and grasped_object_mask header seq: 507310 +stamp: + secs: 1733935226 + nsecs: 352419138 +frame_id: "camera_color_optical_frame" +[INFO] [1733935226.821378]: received mask_obs with header seq: 21905 +stamp: + secs: 1733935226 + nsecs: 825327873 +frame_id: '' and grasped_object_mask header seq: 507317 +stamp: + secs: 1733935226 + nsecs: 586713314 +frame_id: "camera_color_optical_frame" +[INFO] [1733935227.067196]: received mask_obs with header seq: 21906 +stamp: + secs: 1733935227 + nsecs: 71362257 +frame_id: '' and grasped_object_mask header seq: 507324 +stamp: + secs: 1733935226 + nsecs: 821107864 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 101036900 for nearest: 1733935227189209813 - target: 1733935227088172913 at idx 178 +[INFO] [1733935227.321635]: received mask_obs with header seq: 21907 +stamp: + secs: 1733935227 + nsecs: 325707197 +frame_id: '' and grasped_object_mask header seq: 507332 +stamp: + secs: 1733935227 + nsecs: 88172913 +frame_id: "camera_color_optical_frame" +[INFO] [1733935227.569758]: received mask_obs with header seq: 21908 +stamp: + secs: 1733935227 + nsecs: 573556900 +frame_id: '' and grasped_object_mask header seq: 507340 +stamp: + secs: 1733935227 + nsecs: 354316473 +frame_id: "camera_color_optical_frame" +[INFO] [1733935227.820263]: received mask_obs with header seq: 21909 +stamp: + secs: 1733935227 + nsecs: 824471712 +frame_id: '' and grasped_object_mask header seq: 507347 +stamp: + secs: 1733935227 + nsecs: 587174177 +frame_id: "camera_color_optical_frame" +[INFO] [1733935228.067422]: received mask_obs with header seq: 21910 +stamp: + secs: 1733935228 + nsecs: 71466207 +frame_id: '' and grasped_object_mask header seq: 507354 +stamp: + secs: 1733935227 + nsecs: 820017099 +frame_id: "camera_color_optical_frame" +[INFO] [1733935228.321581]: received mask_obs with header seq: 21911 +stamp: + secs: 1733935228 + nsecs: 325696945 +frame_id: '' and grasped_object_mask header seq: 507362 +stamp: + secs: 1733935228 + nsecs: 86137772 +frame_id: "camera_color_optical_frame" +[INFO] [1733935228.568170]: received mask_obs with header seq: 21912 +stamp: + secs: 1733935228 + nsecs: 572247982 +frame_id: '' and grasped_object_mask header seq: 507370 +stamp: + secs: 1733935228 + nsecs: 352271318 +frame_id: "camera_color_optical_frame" +[INFO] [1733935228.822757]: received mask_obs with header seq: 21913 +stamp: + secs: 1733935228 + nsecs: 826850891 +frame_id: '' and grasped_object_mask header seq: 507377 +stamp: + secs: 1733935228 + nsecs: 585121393 +frame_id: "camera_color_optical_frame" +[INFO] [1733935229.078614]: received mask_obs with header seq: 21914 +stamp: + secs: 1733935229 + nsecs: 82789659 +frame_id: '' and grasped_object_mask header seq: 507385 +stamp: + secs: 1733935228 + nsecs: 851215363 +frame_id: "camera_color_optical_frame" +[INFO] [1733935229.336839]: received mask_obs with header seq: 21915 +stamp: + secs: 1733935229 + nsecs: 341068506 +frame_id: '' and grasped_object_mask header seq: 507392 +stamp: + secs: 1733935229 + nsecs: 84069490 +frame_id: "camera_color_optical_frame" +[INFO] [1733935229.592968]: received mask_obs with header seq: 21916 +stamp: + secs: 1733935229 + nsecs: 597051382 +frame_id: '' and grasped_object_mask header seq: 507401 +stamp: + secs: 1733935229 + nsecs: 383467197 +frame_id: "camera_color_optical_frame" +[INFO] [1733935229.858993]: received mask_obs with header seq: 21917 +stamp: + secs: 1733935229 + nsecs: 862663507 +frame_id: '' and grasped_object_mask header seq: 507409 +stamp: + secs: 1733935229 + nsecs: 649575233 +frame_id: "camera_color_optical_frame" +[INFO] [1733935230.123791]: received mask_obs with header seq: 21918 +stamp: + secs: 1733935230 + nsecs: 127400398 +frame_id: '' and grasped_object_mask header seq: 507416 +stamp: + secs: 1733935229 + nsecs: 882402658 +frame_id: "camera_color_optical_frame" +[INFO] [1733935230.393178]: received mask_obs with header seq: 21919 +stamp: + secs: 1733935230 + nsecs: 396697998 +frame_id: '' and grasped_object_mask header seq: 507424 +stamp: + secs: 1733935230 + nsecs: 148535252 +frame_id: "camera_color_optical_frame" +[INFO] [1733935230.659324]: received mask_obs with header seq: 21920 +stamp: + secs: 1733935230 + nsecs: 662387132 +frame_id: '' and grasped_object_mask header seq: 507433 +stamp: + secs: 1733935230 + nsecs: 447934389 +frame_id: "camera_color_optical_frame" +[INFO] [1733935230.922837]: received mask_obs with header seq: 21921 +stamp: + secs: 1733935230 + nsecs: 926246404 +frame_id: '' and grasped_object_mask header seq: 507441 +stamp: + secs: 1733935230 + nsecs: 714044333 +frame_id: "camera_color_optical_frame" +[INFO] [1733935231.182338]: received mask_obs with header seq: 21922 +stamp: + secs: 1733935231 + nsecs: 186518192 +frame_id: '' and grasped_object_mask header seq: 507448 +stamp: + secs: 1733935230 + nsecs: 946874380 +frame_id: "camera_color_optical_frame" +[INFO] [1733935231.440493]: received mask_obs with header seq: 21923 +stamp: + secs: 1733935231 + nsecs: 444426059 +frame_id: '' and grasped_object_mask header seq: 507456 +stamp: + secs: 1733935231 + nsecs: 213035345 +frame_id: "camera_color_optical_frame" +[INFO] [1733935231.678290]: received mask_obs with header seq: 21924 +stamp: + secs: 1733935231 + nsecs: 682198524 +frame_id: '' and grasped_object_mask header seq: 507463 +stamp: + secs: 1733935231 + nsecs: 445910454 +frame_id: "camera_color_optical_frame" +[INFO] [1733935231.923566]: received mask_obs with header seq: 21925 +stamp: + secs: 1733935231 + nsecs: 926250696 +frame_id: '' and grasped_object_mask header seq: 507471 +stamp: + secs: 1733935231 + nsecs: 712033987 +frame_id: "camera_color_optical_frame" +[INFO] [1733935232.182889]: received mask_obs with header seq: 21926 +stamp: + secs: 1733935232 + nsecs: 187014102 +frame_id: '' and grasped_object_mask header seq: 507478 +stamp: + secs: 1733935231 + nsecs: 944874763 +frame_id: "camera_color_optical_frame" +[INFO] [1733935232.424132]: received mask_obs with header seq: 21927 +stamp: + secs: 1733935232 + nsecs: 426733016 +frame_id: '' and grasped_object_mask header seq: 507486 +stamp: + secs: 1733935232 + nsecs: 211043596 +frame_id: "camera_color_optical_frame" +[INFO] [1733935232.695210]: received mask_obs with header seq: 21928 +stamp: + secs: 1733935232 + nsecs: 698362588 +frame_id: '' and grasped_object_mask header seq: 507493 +stamp: + secs: 1733935232 + nsecs: 443910837 +frame_id: "camera_color_optical_frame" +[INFO] [1733935232.957511]: received mask_obs with header seq: 21929 +stamp: + secs: 1733935232 + nsecs: 961392641 +frame_id: '' and grasped_object_mask header seq: 507502 +stamp: + secs: 1733935232 + nsecs: 743289709 +frame_id: "camera_color_optical_frame" +[INFO] [1733935233.220049]: received mask_obs with header seq: 21930 +stamp: + secs: 1733935233 + nsecs: 224112033 +frame_id: '' and grasped_object_mask header seq: 507510 +stamp: + secs: 1733935233 + nsecs: 9395838 +frame_id: "camera_color_optical_frame" +[INFO] [1733935233.472825]: received mask_obs with header seq: 21931 +stamp: + secs: 1733935233 + nsecs: 476829767 +frame_id: '' and grasped_object_mask header seq: 507517 +stamp: + secs: 1733935233 + nsecs: 242276907 +frame_id: "camera_color_optical_frame" +[INFO] [1733935233.730363]: received mask_obs with header seq: 21932 +stamp: + secs: 1733935233 + nsecs: 734474658 +frame_id: '' and grasped_object_mask header seq: 507525 +stamp: + secs: 1733935233 + nsecs: 508406878 +frame_id: "camera_color_optical_frame" +[INFO] [1733935233.989283]: received mask_obs with header seq: 21933 +stamp: + secs: 1733935233 + nsecs: 993208646 +frame_id: '' and grasped_object_mask header seq: 507533 +stamp: + secs: 1733935233 + nsecs: 774516821 +frame_id: "camera_color_optical_frame" +[INFO] [1733935234.252578]: received mask_obs with header seq: 21934 +stamp: + secs: 1733935234 + nsecs: 256666660 +frame_id: '' and grasped_object_mask header seq: 507541 +stamp: + secs: 1733935234 + nsecs: 40629387 +frame_id: "camera_color_optical_frame" +[INFO] [1733935234.509777]: received mask_obs with header seq: 21935 +stamp: + secs: 1733935234 + nsecs: 513907909 +frame_id: '' and grasped_object_mask header seq: 507548 +stamp: + secs: 1733935234 + nsecs: 273503780 +frame_id: "camera_color_optical_frame" +[INFO] [1733935234.770283]: received mask_obs with header seq: 21936 +stamp: + secs: 1733935234 + nsecs: 774463176 +frame_id: '' and grasped_object_mask header seq: 507556 +stamp: + secs: 1733935234 + nsecs: 539627790 +frame_id: "camera_color_optical_frame" +[INFO] [1733935235.029457]: received mask_obs with header seq: 21937 +stamp: + secs: 1733935235 + nsecs: 33633947 +frame_id: '' and grasped_object_mask header seq: 507564 +stamp: + secs: 1733935234 + nsecs: 805729866 +frame_id: "camera_color_optical_frame" +[INFO] [1733935235.269657]: received mask_obs with header seq: 21938 +stamp: + secs: 1733935235 + nsecs: 273810625 +frame_id: '' and grasped_object_mask header seq: 507571 +stamp: + secs: 1733935235 + nsecs: 38573980 +frame_id: "camera_color_optical_frame" +[INFO] [1733935235.531607]: received mask_obs with header seq: 21939 +stamp: + secs: 1733935235 + nsecs: 535701036 +frame_id: '' and grasped_object_mask header seq: 507579 +stamp: + secs: 1733935235 + nsecs: 304711342 +frame_id: "camera_color_optical_frame" +[INFO] [1733935235.769487]: received mask_obs with header seq: 21940 +stamp: + secs: 1733935235 + nsecs: 773558855 +frame_id: '' and grasped_object_mask header seq: 507586 +stamp: + secs: 1733935235 + nsecs: 537563086 +frame_id: "camera_color_optical_frame" +[INFO] [1733935236.010330]: received mask_obs with header seq: 21941 +stamp: + secs: 1733935236 + nsecs: 14442682 +frame_id: '' and grasped_object_mask header seq: 507593 +stamp: + secs: 1733935235 + nsecs: 770399094 +frame_id: "camera_color_optical_frame" +[INFO] [1733935236.262984]: received mask_obs with header seq: 21942 +stamp: + secs: 1733935236 + nsecs: 267210006 +frame_id: '' and grasped_object_mask header seq: 507601 +stamp: + secs: 1733935236 + nsecs: 36499977 +frame_id: "camera_color_optical_frame" +[INFO] [1733935236.531481]: received mask_obs with header seq: 21943 +stamp: + secs: 1733935236 + nsecs: 534975290 +frame_id: '' and grasped_object_mask header seq: 507609 +stamp: + secs: 1733935236 + nsecs: 302633524 +frame_id: "camera_color_optical_frame" +[INFO] [1733935236.789376]: received mask_obs with header seq: 21944 +stamp: + secs: 1733935236 + nsecs: 793395757 +frame_id: '' and grasped_object_mask header seq: 507617 +stamp: + secs: 1733935236 + nsecs: 568746328 +frame_id: "camera_color_optical_frame" +[INFO] [1733935237.031896]: received mask_obs with header seq: 21945 +stamp: + secs: 1733935237 + nsecs: 35560131 +frame_id: '' and grasped_object_mask header seq: 507624 +stamp: + secs: 1733935236 + nsecs: 801575899 +frame_id: "camera_color_optical_frame" +[INFO] [1733935237.290106]: received mask_obs with header seq: 21946 +stamp: + secs: 1733935237 + nsecs: 294316530 +frame_id: '' and grasped_object_mask header seq: 507632 +stamp: + secs: 1733935237 + nsecs: 67682266 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -107551737 for nearest: 1733935237226260977 - target: 1733935237333812714 at idx 249 +[INFO] [1733935237.550809]: received mask_obs with header seq: 21947 +stamp: + secs: 1733935237 + nsecs: 554226636 +frame_id: '' and grasped_object_mask header seq: 507640 +stamp: + secs: 1733935237 + nsecs: 333812714 +frame_id: "camera_color_optical_frame" +[INFO] [1733935237.811553]: received mask_obs with header seq: 21948 +stamp: + secs: 1733935237 + nsecs: 815675973 +frame_id: '' and grasped_object_mask header seq: 507647 +stamp: + secs: 1733935237 + nsecs: 566658497 +frame_id: "camera_color_optical_frame" +[INFO] [1733935238.068314]: received mask_obs with header seq: 21949 +stamp: + secs: 1733935238 + nsecs: 72452545 +frame_id: '' and grasped_object_mask header seq: 507655 +stamp: + secs: 1733935237 + nsecs: 832749128 +frame_id: "camera_color_optical_frame" +[INFO] [1733935238.329026]: received mask_obs with header seq: 21950 +stamp: + secs: 1733935238 + nsecs: 331978797 +frame_id: '' and grasped_object_mask header seq: 507663 +stamp: + secs: 1733935238 + nsecs: 98864317 +frame_id: "camera_color_optical_frame" +[INFO] [1733935238.567165]: received mask_obs with header seq: 21951 +stamp: + secs: 1733935238 + nsecs: 571337461 +frame_id: '' and grasped_object_mask header seq: 507670 +stamp: + secs: 1733935238 + nsecs: 331732512 +frame_id: "camera_color_optical_frame" +[INFO] [1733935238.823507]: received mask_obs with header seq: 21952 +stamp: + secs: 1733935238 + nsecs: 827619075 +frame_id: '' and grasped_object_mask header seq: 507678 +stamp: + secs: 1733935238 + nsecs: 597847223 +frame_id: "camera_color_optical_frame" +[INFO] [1733935239.066957]: received mask_obs with header seq: 21953 +stamp: + secs: 1733935239 + nsecs: 71071863 +frame_id: '' and grasped_object_mask header seq: 507685 +stamp: + secs: 1733935238 + nsecs: 830679893 +frame_id: "camera_color_optical_frame" +[INFO] [1733935239.323463]: received mask_obs with header seq: 21954 +stamp: + secs: 1733935239 + nsecs: 327624797 +frame_id: '' and grasped_object_mask header seq: 507693 +stamp: + secs: 1733935239 + nsecs: 96801996 +frame_id: "camera_color_optical_frame" +[INFO] [1733935239.565878]: received mask_obs with header seq: 21955 +stamp: + secs: 1733935239 + nsecs: 569805860 +frame_id: '' and grasped_object_mask header seq: 507700 +stamp: + secs: 1733935239 + nsecs: 329674482 +frame_id: "camera_color_optical_frame" +[INFO] [1733935239.821616]: received mask_obs with header seq: 21956 +stamp: + secs: 1733935239 + nsecs: 825678825 +frame_id: '' and grasped_object_mask header seq: 507708 +stamp: + secs: 1733935239 + nsecs: 595797062 +frame_id: "camera_color_optical_frame" +[INFO] [1733935240.085332]: received mask_obs with header seq: 21957 +stamp: + secs: 1733935240 + nsecs: 88120460 +frame_id: '' and grasped_object_mask header seq: 507716 +stamp: + secs: 1733935239 + nsecs: 861897945 +frame_id: "camera_color_optical_frame" +[INFO] [1733935240.343183]: received mask_obs with header seq: 21958 +stamp: + secs: 1733935240 + nsecs: 347238063 +frame_id: '' and grasped_object_mask header seq: 507723 +stamp: + secs: 1733935240 + nsecs: 95680237 +frame_id: "camera_color_optical_frame" +[INFO] [1733935240.599689]: received mask_obs with header seq: 21959 +stamp: + secs: 1733935240 + nsecs: 603528738 +frame_id: '' and grasped_object_mask header seq: 507731 +stamp: + secs: 1733935240 + nsecs: 363653660 +frame_id: "camera_color_optical_frame" +[INFO] [1733935240.858874]: received mask_obs with header seq: 21960 +stamp: + secs: 1733935240 + nsecs: 863046407 +frame_id: '' and grasped_object_mask header seq: 507739 +stamp: + secs: 1733935240 + nsecs: 631783009 +frame_id: "camera_color_optical_frame" +[INFO] [1733935241.116060]: received mask_obs with header seq: 21961 +stamp: + secs: 1733935241 + nsecs: 120050191 +frame_id: '' and grasped_object_mask header seq: 507746 +stamp: + secs: 1733935240 + nsecs: 866524458 +frame_id: "camera_color_optical_frame" +[INFO] [1733935241.383276]: received mask_obs with header seq: 21962 +stamp: + secs: 1733935241 + nsecs: 387110948 +frame_id: '' and grasped_object_mask header seq: 507754 +stamp: + secs: 1733935241 + nsecs: 133625746 +frame_id: "camera_color_optical_frame" +[INFO] [1733935241.623018]: received mask_obs with header seq: 21963 +stamp: + secs: 1733935241 + nsecs: 626954793 +frame_id: '' and grasped_object_mask header seq: 507762 +stamp: + secs: 1733935241 + nsecs: 400085926 +frame_id: "camera_color_optical_frame" +[INFO] [1733935241.863218]: received mask_obs with header seq: 21964 +stamp: + secs: 1733935241 + nsecs: 867071151 +frame_id: '' and grasped_object_mask header seq: 507769 +stamp: + secs: 1733935241 + nsecs: 633239508 +frame_id: "camera_color_optical_frame" +[INFO] [1733935242.100084]: received mask_obs with header seq: 21965 +stamp: + secs: 1733935242 + nsecs: 104247808 +frame_id: '' and grasped_object_mask header seq: 507776 +stamp: + secs: 1733935241 + nsecs: 866392851 +frame_id: "camera_color_optical_frame" +[INFO] [1733935242.359636]: received mask_obs with header seq: 21966 +stamp: + secs: 1733935242 + nsecs: 363789796 +frame_id: '' and grasped_object_mask header seq: 507784 +stamp: + secs: 1733935242 + nsecs: 132843494 +frame_id: "camera_color_optical_frame" +[INFO] [1733935242.627135]: received mask_obs with header seq: 21967 +stamp: + secs: 1733935242 + nsecs: 631369113 +frame_id: '' and grasped_object_mask header seq: 507792 +stamp: + secs: 1733935242 + nsecs: 399286509 +frame_id: "camera_color_optical_frame" +[INFO] [1733935242.884587]: received mask_obs with header seq: 21968 +stamp: + secs: 1733935242 + nsecs: 888249635 +frame_id: '' and grasped_object_mask header seq: 507800 +stamp: + secs: 1733935242 + nsecs: 665729284 +frame_id: "camera_color_optical_frame" +[INFO] [1733935243.129335]: received mask_obs with header seq: 21969 +stamp: + secs: 1733935243 + nsecs: 133538007 +frame_id: '' and grasped_object_mask header seq: 507807 +stamp: + secs: 1733935242 + nsecs: 898866653 +frame_id: "camera_color_optical_frame" +[INFO] [1733935243.389655]: received mask_obs with header seq: 21970 +stamp: + secs: 1733935243 + nsecs: 393302679 +frame_id: '' and grasped_object_mask header seq: 507815 +stamp: + secs: 1733935243 + nsecs: 165317059 +frame_id: "camera_color_optical_frame" +[INFO] [1733935243.615351]: received mask_obs with header seq: 21971 +stamp: + secs: 1733935243 + nsecs: 618676185 +frame_id: '' and grasped_object_mask header seq: 507822 +stamp: + secs: 1733935243 + nsecs: 398463964 +frame_id: "camera_color_optical_frame" +[INFO] [1733935243.853235]: received mask_obs with header seq: 21972 +stamp: + secs: 1733935243 + nsecs: 856332063 +frame_id: '' and grasped_object_mask header seq: 507829 +stamp: + secs: 1733935243 + nsecs: 631609917 +frame_id: "camera_color_optical_frame" +[INFO] [1733935244.115043]: received mask_obs with header seq: 21973 +stamp: + secs: 1733935244 + nsecs: 118335962 +frame_id: '' and grasped_object_mask header seq: 507837 +stamp: + secs: 1733935243 + nsecs: 898064137 +frame_id: "camera_color_optical_frame" +[INFO] [1733935244.377846]: received mask_obs with header seq: 21974 +stamp: + secs: 1733935244 + nsecs: 381880044 +frame_id: '' and grasped_object_mask header seq: 507845 +stamp: + secs: 1733935244 + nsecs: 164513588 +frame_id: "camera_color_optical_frame" +[INFO] [1733935244.632085]: received mask_obs with header seq: 21975 +stamp: + secs: 1733935244 + nsecs: 636172056 +frame_id: '' and grasped_object_mask header seq: 507852 +stamp: + secs: 1733935244 + nsecs: 397656441 +frame_id: "camera_color_optical_frame" +[INFO] [1733935244.873886]: received mask_obs with header seq: 21976 +stamp: + secs: 1733935244 + nsecs: 877926111 +frame_id: '' and grasped_object_mask header seq: 507859 +stamp: + secs: 1733935244 + nsecs: 630798101 +frame_id: "camera_color_optical_frame" +[INFO] [1733935245.129919]: received mask_obs with header seq: 21977 +stamp: + secs: 1733935245 + nsecs: 133929252 +frame_id: '' and grasped_object_mask header seq: 507867 +stamp: + secs: 1733935244 + nsecs: 897246361 +frame_id: "camera_color_optical_frame" +[INFO] [1733935245.385835]: received mask_obs with header seq: 21978 +stamp: + secs: 1733935245 + nsecs: 389966011 +frame_id: '' and grasped_object_mask header seq: 507875 +stamp: + secs: 1733935245 + nsecs: 163698912 +frame_id: "camera_color_optical_frame" +[INFO] [1733935245.627415]: received mask_obs with header seq: 21979 +stamp: + secs: 1733935245 + nsecs: 631128311 +frame_id: '' and grasped_object_mask header seq: 507882 +stamp: + secs: 1733935245 + nsecs: 396844625 +frame_id: "camera_color_optical_frame" +[INFO] [1733935245.868954]: received mask_obs with header seq: 21980 +stamp: + secs: 1733935245 + nsecs: 873158693 +frame_id: '' and grasped_object_mask header seq: 507889 +stamp: + secs: 1733935245 + nsecs: 629991531 +frame_id: "camera_color_optical_frame" +[INFO] [1733935246.126707]: received mask_obs with header seq: 21981 +stamp: + secs: 1733935246 + nsecs: 130706071 +frame_id: '' and grasped_object_mask header seq: 507897 +stamp: + secs: 1733935245 + nsecs: 896444559 +frame_id: "camera_color_optical_frame" +[INFO] [1733935246.381882]: received mask_obs with header seq: 21982 +stamp: + secs: 1733935246 + nsecs: 385669469 +frame_id: '' and grasped_object_mask header seq: 507904 +stamp: + secs: 1733935246 + nsecs: 129592419 +frame_id: "camera_color_optical_frame" +[INFO] [1733935246.643546]: received mask_obs with header seq: 21983 +stamp: + secs: 1733935246 + nsecs: 646885633 +frame_id: '' and grasped_object_mask header seq: 507913 +stamp: + secs: 1733935246 + nsecs: 429354429 +frame_id: "camera_color_optical_frame" +[INFO] [1733935246.897732]: received mask_obs with header seq: 21984 +stamp: + secs: 1733935246 + nsecs: 901472091 +frame_id: '' and grasped_object_mask header seq: 507920 +stamp: + secs: 1733935246 + nsecs: 662502289 +frame_id: "camera_color_optical_frame" +[INFO] [1733935247.142960]: received mask_obs with header seq: 21985 +stamp: + secs: 1733935247 + nsecs: 146862268 +frame_id: '' and grasped_object_mask header seq: 507928 +stamp: + secs: 1733935246 + nsecs: 928958178 +frame_id: "camera_color_optical_frame" +[INFO] [1733935247.397418]: received mask_obs with header seq: 21986 +stamp: + secs: 1733935247 + nsecs: 401607275 +frame_id: '' and grasped_object_mask header seq: 507935 +stamp: + secs: 1733935247 + nsecs: 162103415 +frame_id: "camera_color_optical_frame" +[INFO] [1733935247.645539]: received mask_obs with header seq: 21987 +stamp: + secs: 1733935247 + nsecs: 648052453 +frame_id: '' and grasped_object_mask header seq: 507943 +stamp: + secs: 1733935247 + nsecs: 428555727 +frame_id: "camera_color_optical_frame" +[INFO] [1733935247.903908]: received mask_obs with header seq: 21988 +stamp: + secs: 1733935247 + nsecs: 908102512 +frame_id: '' and grasped_object_mask header seq: 507950 +stamp: + secs: 1733935247 + nsecs: 661702156 +frame_id: "camera_color_optical_frame" +[INFO] [1733935248.161847]: received mask_obs with header seq: 21989 +stamp: + secs: 1733935248 + nsecs: 165865421 +frame_id: '' and grasped_object_mask header seq: 507958 +stamp: + secs: 1733935247 + nsecs: 928155184 +frame_id: "camera_color_optical_frame" +[INFO] [1733935248.414460]: received mask_obs with header seq: 21990 +stamp: + secs: 1733935248 + nsecs: 417654275 +frame_id: '' and grasped_object_mask header seq: 507966 +stamp: + secs: 1733935248 + nsecs: 194614649 +frame_id: "camera_color_optical_frame" +[INFO] [1733935248.655082]: received mask_obs with header seq: 21991 +stamp: + secs: 1733935248 + nsecs: 659236907 +frame_id: '' and grasped_object_mask header seq: 507973 +stamp: + secs: 1733935248 + nsecs: 427766562 +frame_id: "camera_color_optical_frame" +[INFO] [1733935248.904914]: received mask_obs with header seq: 21992 +stamp: + secs: 1733935248 + nsecs: 909098863 +frame_id: '' and grasped_object_mask header seq: 507980 +stamp: + secs: 1733935248 + nsecs: 660919905 +frame_id: "camera_color_optical_frame" +[INFO] [1733935249.005308]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1733935249.163340]: received mask_obs with header seq: 21993 +stamp: + secs: 1733935249 + nsecs: 167230606 +frame_id: '' and grasped_object_mask header seq: 507988 +stamp: + secs: 1733935248 + nsecs: 927380800 +frame_id: "camera_color_optical_frame" +[INFO] [1733935249.800464]: Stored episode 4. +[INFO] [1733935249.812378]: Saving video... +[INFO] [1733935250.031466]: Video saved! +Starting episode 4 +[INFO] [1733935250.101843]: resetting environment +[INFO] [1733935250.129043]: cleared current plan +[INFO] [1733935250.136156]: moving to home +[INFO] [1733935252.946371]: reached home +[INFO] [1733935252.956563]: reset action history +[INFO] [1733935255.752957]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1733935260.171773]: resetting environment +[INFO] [1733935260.206949]: cleared current plan +[INFO] [1733935260.207552]: moving to home +[INFO] [1733935261.211303]: reached home +[INFO] [1733935261.213462]: reset action history +[INFO] [1733935264.009138]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/episode_rosbags/episode_4_2024-12-11-11-41-04.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-743294.log +started roslaunch server http://158.130.50.37:33161/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [744203] +process[print_text-14]: started with pid [744204] +process[rosbag_record-15]: started with pid [744205] +started bagging! +[INFO] [1733935265.870902]: received mask_obs with header seq: 21994 +stamp: + secs: 1733935265 + nsecs: 874448537 +frame_id: '' and grasped_object_mask header seq: 508491 +stamp: + secs: 1733935265 + nsecs: 680428982 +frame_id: "camera_color_optical_frame" +[INFO] [1733935266.087657]: received mask_obs with header seq: 21995 +stamp: + secs: 1733935266 + nsecs: 91390371 +frame_id: '' and grasped_object_mask header seq: 508497 +stamp: + secs: 1733935265 + nsecs: 880278587 +frame_id: "camera_color_optical_frame" +[INFO] [1733935266.312902]: received mask_obs with header seq: 21996 +stamp: + secs: 1733935266 + nsecs: 316622257 +frame_id: '' and grasped_object_mask header seq: 508504 +stamp: + secs: 1733935266 + nsecs: 113433123 +frame_id: "camera_color_optical_frame" +[INFO] [1733935266.552055]: received mask_obs with header seq: 21997 +stamp: + secs: 1733935266 + nsecs: 556087493 +frame_id: '' and grasped_object_mask header seq: 508511 +stamp: + secs: 1733935266 + nsecs: 346585989 +frame_id: "camera_color_optical_frame" +[INFO] [1733935266.800626]: received mask_obs with header seq: 21998 +stamp: + secs: 1733935266 + nsecs: 804750919 +frame_id: '' and grasped_object_mask header seq: 508518 +stamp: + secs: 1733935266 + nsecs: 579738140 +frame_id: "camera_color_optical_frame" +[INFO] [1733935267.043798]: received mask_obs with header seq: 21999 +stamp: + secs: 1733935267 + nsecs: 47749519 +frame_id: '' and grasped_object_mask header seq: 508525 +stamp: + secs: 1733935266 + nsecs: 812891245 +frame_id: "camera_color_optical_frame" +[INFO] [1733935267.279517]: received mask_obs with header seq: 22000 +stamp: + secs: 1733935267 + nsecs: 283618211 +frame_id: '' and grasped_object_mask header seq: 508532 +stamp: + secs: 1733935267 + nsecs: 46042204 +frame_id: "camera_color_optical_frame" +[INFO] [1733935267.525679]: received mask_obs with header seq: 22001 +stamp: + secs: 1733935267 + nsecs: 529753446 +frame_id: '' and grasped_object_mask header seq: 508540 +stamp: + secs: 1733935267 + nsecs: 312495947 +frame_id: "camera_color_optical_frame" +[INFO] [1733935267.798302]: received mask_obs with header seq: 22002 +stamp: + secs: 1733935267 + nsecs: 802314281 +frame_id: '' and grasped_object_mask header seq: 508547 +stamp: + secs: 1733935267 + nsecs: 545643568 +frame_id: "camera_color_optical_frame" +[INFO] [1733935268.061790]: received mask_obs with header seq: 22003 +stamp: + secs: 1733935268 + nsecs: 63741922 +frame_id: '' and grasped_object_mask header seq: 508556 +stamp: + secs: 1733935267 + nsecs: 845404625 +frame_id: "camera_color_optical_frame" +[INFO] [1733935268.318586]: received mask_obs with header seq: 22004 +stamp: + secs: 1733935268 + nsecs: 322679042 +frame_id: '' and grasped_object_mask header seq: 508563 +stamp: + secs: 1733935268 + nsecs: 78555107 +frame_id: "camera_color_optical_frame" +[INFO] [1733935268.575917]: received mask_obs with header seq: 22005 +stamp: + secs: 1733935268 + nsecs: 580078840 +frame_id: '' and grasped_object_mask header seq: 508571 +stamp: + secs: 1733935268 + nsecs: 345017433 +frame_id: "camera_color_optical_frame" +[INFO] [1733935268.820513]: received mask_obs with header seq: 22006 +stamp: + secs: 1733935268 + nsecs: 824342966 +frame_id: '' and grasped_object_mask header seq: 508578 +stamp: + secs: 1733935268 + nsecs: 578172684 +frame_id: "camera_color_optical_frame" +[INFO] [1733935269.078764]: received mask_obs with header seq: 22007 +stamp: + secs: 1733935269 + nsecs: 82834005 +frame_id: '' and grasped_object_mask header seq: 508586 +stamp: + secs: 1733935268 + nsecs: 844635487 +frame_id: "camera_color_optical_frame" +[INFO] [1733935269.333668]: received mask_obs with header seq: 22008 +stamp: + secs: 1733935269 + nsecs: 337360858 +frame_id: '' and grasped_object_mask header seq: 508594 +stamp: + secs: 1733935269 + nsecs: 111094952 +frame_id: "camera_color_optical_frame" +[INFO] [1733935269.576254]: received mask_obs with header seq: 22009 +stamp: + secs: 1733935269 + nsecs: 580028057 +frame_id: '' and grasped_object_mask header seq: 508601 +stamp: + secs: 1733935269 + nsecs: 344243526 +frame_id: "camera_color_optical_frame" +[INFO] [1733935269.835849]: received mask_obs with header seq: 22010 +stamp: + secs: 1733935269 + nsecs: 839873552 +frame_id: '' and grasped_object_mask header seq: 508609 +stamp: + secs: 1733935269 + nsecs: 610699177 +frame_id: "camera_color_optical_frame" +[INFO] [1733935270.096721]: received mask_obs with header seq: 22011 +stamp: + secs: 1733935270 + nsecs: 100509881 +frame_id: '' and grasped_object_mask header seq: 508617 +stamp: + secs: 1733935269 + nsecs: 877156258 +frame_id: "camera_color_optical_frame" +[INFO] [1733935270.360034]: received mask_obs with header seq: 22012 +stamp: + secs: 1733935270 + nsecs: 363539218 +frame_id: '' and grasped_object_mask header seq: 508625 +stamp: + secs: 1733935270 + nsecs: 143611193 +frame_id: "camera_color_optical_frame" +[INFO] [1733935270.633063]: received mask_obs with header seq: 22013 +stamp: + secs: 1733935270 + nsecs: 636982440 +frame_id: '' and grasped_object_mask header seq: 508632 +stamp: + secs: 1733935270 + nsecs: 376759291 +frame_id: "camera_color_optical_frame" +[INFO] [1733935270.896601]: received mask_obs with header seq: 22014 +stamp: + secs: 1733935270 + nsecs: 900786876 +frame_id: '' and grasped_object_mask header seq: 508641 +stamp: + secs: 1733935270 + nsecs: 676519871 +frame_id: "camera_color_optical_frame" +[INFO] [1733935271.156164]: received mask_obs with header seq: 22015 +stamp: + secs: 1733935271 + nsecs: 159986019 +frame_id: '' and grasped_object_mask header seq: 508649 +stamp: + secs: 1733935270 + nsecs: 942975044 +frame_id: "camera_color_optical_frame" +[INFO] [1733935271.426388]: received mask_obs with header seq: 22016 +stamp: + secs: 1733935271 + nsecs: 428913354 +frame_id: '' and grasped_object_mask header seq: 508656 +stamp: + secs: 1733935271 + nsecs: 176124811 +frame_id: "camera_color_optical_frame" +[INFO] [1733935271.676886]: received mask_obs with header seq: 22017 +stamp: + secs: 1733935271 + nsecs: 681032657 +frame_id: '' and grasped_object_mask header seq: 508664 +stamp: + secs: 1733935271 + nsecs: 442582369 +frame_id: "camera_color_optical_frame" +[INFO] [1733935271.924970]: received mask_obs with header seq: 22018 +stamp: + secs: 1733935271 + nsecs: 928612709 +frame_id: '' and grasped_object_mask header seq: 508672 +stamp: + secs: 1733935271 + nsecs: 709040880 +frame_id: "camera_color_optical_frame" +[INFO] [1733935272.179274]: received mask_obs with header seq: 22019 +stamp: + secs: 1733935272 + nsecs: 183026313 +frame_id: '' and grasped_object_mask header seq: 508679 +stamp: + secs: 1733935271 + nsecs: 942190647 +frame_id: "camera_color_optical_frame" +[INFO] [1733935272.445122]: received mask_obs with header seq: 22020 +stamp: + secs: 1733935272 + nsecs: 449256420 +frame_id: '' and grasped_object_mask header seq: 508687 +stamp: + secs: 1733935272 + nsecs: 208651543 +frame_id: "camera_color_optical_frame" +[INFO] [1733935272.706902]: received mask_obs with header seq: 22021 +stamp: + secs: 1733935272 + nsecs: 711053371 +frame_id: '' and grasped_object_mask header seq: 508695 +stamp: + secs: 1733935272 + nsecs: 475111961 +frame_id: "camera_color_optical_frame" +[INFO] [1733935272.963375]: received mask_obs with header seq: 22022 +stamp: + secs: 1733935272 + nsecs: 967496395 +frame_id: '' and grasped_object_mask header seq: 508703 +stamp: + secs: 1733935272 + nsecs: 741573095 +frame_id: "camera_color_optical_frame" +[INFO] [1733935273.226444]: received mask_obs with header seq: 22023 +stamp: + secs: 1733935273 + nsecs: 230534315 +frame_id: '' and grasped_object_mask header seq: 508711 +stamp: + secs: 1733935273 + nsecs: 8032799 +frame_id: "camera_color_optical_frame" +[INFO] [1733935273.497434]: received mask_obs with header seq: 22024 +stamp: + secs: 1733935273 + nsecs: 501358270 +frame_id: '' and grasped_object_mask header seq: 508718 +stamp: + secs: 1733935273 + nsecs: 241180897 +frame_id: "camera_color_optical_frame" +[INFO] [1733935273.765539]: received mask_obs with header seq: 22025 +stamp: + secs: 1733935273 + nsecs: 769543886 +frame_id: '' and grasped_object_mask header seq: 508727 +stamp: + secs: 1733935273 + nsecs: 540942669 +frame_id: "camera_color_optical_frame" +[INFO] [1733935274.026568]: received mask_obs with header seq: 22026 +stamp: + secs: 1733935274 + nsecs: 30534505 +frame_id: '' and grasped_object_mask header seq: 508735 +stamp: + secs: 1733935273 + nsecs: 807397127 +frame_id: "camera_color_optical_frame" +[INFO] [1733935274.279404]: received mask_obs with header seq: 22027 +stamp: + secs: 1733935274 + nsecs: 283306121 +frame_id: '' and grasped_object_mask header seq: 508742 +stamp: + secs: 1733935274 + nsecs: 40544748 +frame_id: "camera_color_optical_frame" +[INFO] [1733935274.525391]: received mask_obs with header seq: 22028 +stamp: + secs: 1733935274 + nsecs: 528836011 +frame_id: '' and grasped_object_mask header seq: 508750 +stamp: + secs: 1733935274 + nsecs: 306996346 +frame_id: "camera_color_optical_frame" +[INFO] [1733935274.788214]: received mask_obs with header seq: 22029 +stamp: + secs: 1733935274 + nsecs: 792100906 +frame_id: '' and grasped_object_mask header seq: 508757 +stamp: + secs: 1733935274 + nsecs: 540141106 +frame_id: "camera_color_optical_frame" +[INFO] [1733935275.055091]: received mask_obs with header seq: 22030 +stamp: + secs: 1733935275 + nsecs: 58726549 +frame_id: '' and grasped_object_mask header seq: 508765 +stamp: + secs: 1733935274 + nsecs: 806593657 +frame_id: "camera_color_optical_frame" +[INFO] [1733935275.313159]: received mask_obs with header seq: 22031 +stamp: + secs: 1733935275 + nsecs: 317296028 +frame_id: '' and grasped_object_mask header seq: 508773 +stamp: + secs: 1733935275 + nsecs: 73042631 +frame_id: "camera_color_optical_frame" +[INFO] [1733935275.571675]: received mask_obs with header seq: 22032 +stamp: + secs: 1733935275 + nsecs: 575808048 +frame_id: '' and grasped_object_mask header seq: 508781 +stamp: + secs: 1733935275 + nsecs: 339486599 +frame_id: "camera_color_optical_frame" +[INFO] [1733935275.811702]: received mask_obs with header seq: 22033 +stamp: + secs: 1733935275 + nsecs: 815825462 +frame_id: '' and grasped_object_mask header seq: 508788 +stamp: + secs: 1733935275 + nsecs: 572624683 +frame_id: "camera_color_optical_frame" +[INFO] [1733935276.055387]: received mask_obs with header seq: 22034 +stamp: + secs: 1733935276 + nsecs: 59267520 +frame_id: '' and grasped_object_mask header seq: 508796 +stamp: + secs: 1733935275 + nsecs: 839067221 +frame_id: "camera_color_optical_frame" +[INFO] [1733935276.315647]: received mask_obs with header seq: 22035 +stamp: + secs: 1733935276 + nsecs: 319507122 +frame_id: '' and grasped_object_mask header seq: 508803 +stamp: + secs: 1733935276 + nsecs: 72205782 +frame_id: "camera_color_optical_frame" +[INFO] [1733935276.573860]: received mask_obs with header seq: 22036 +stamp: + secs: 1733935276 + nsecs: 577664613 +frame_id: '' and grasped_object_mask header seq: 508811 +stamp: + secs: 1733935276 + nsecs: 338650942 +frame_id: "camera_color_optical_frame" +[INFO] [1733935276.834054]: received mask_obs with header seq: 22037 +stamp: + secs: 1733935276 + nsecs: 837839841 +frame_id: '' and grasped_object_mask header seq: 508819 +stamp: + secs: 1733935276 + nsecs: 605095148 +frame_id: "camera_color_optical_frame" +[INFO] [1733935277.073548]: received mask_obs with header seq: 22038 +stamp: + secs: 1733935277 + nsecs: 76867341 +frame_id: '' and grasped_object_mask header seq: 508826 +stamp: + secs: 1733935276 + nsecs: 838234186 +frame_id: "camera_color_optical_frame" +[INFO] [1733935277.330727]: received mask_obs with header seq: 22039 +stamp: + secs: 1733935277 + nsecs: 334844112 +frame_id: '' and grasped_object_mask header seq: 508834 +stamp: + secs: 1733935277 + nsecs: 104685068 +frame_id: "camera_color_optical_frame" +[INFO] [1733935277.588380]: received mask_obs with header seq: 22040 +stamp: + secs: 1733935277 + nsecs: 592323064 +frame_id: '' and grasped_object_mask header seq: 508841 +stamp: + secs: 1733935277 + nsecs: 337836981 +frame_id: "camera_color_optical_frame" +[INFO] [1733935277.843155]: received mask_obs with header seq: 22041 +stamp: + secs: 1733935277 + nsecs: 847077131 +frame_id: '' and grasped_object_mask header seq: 508849 +stamp: + secs: 1733935277 + nsecs: 604295969 +frame_id: "camera_color_optical_frame" +[INFO] [1733935278.102921]: received mask_obs with header seq: 22042 +stamp: + secs: 1733935278 + nsecs: 107098579 +frame_id: '' and grasped_object_mask header seq: 508857 +stamp: + secs: 1733935277 + nsecs: 870754957 +frame_id: "camera_color_optical_frame" +[INFO] [1733935278.358699]: received mask_obs with header seq: 22043 +stamp: + secs: 1733935278 + nsecs: 362772941 +frame_id: '' and grasped_object_mask header seq: 508865 +stamp: + secs: 1733935278 + nsecs: 137207031 +frame_id: "camera_color_optical_frame" +[INFO] [1733935278.603505]: received mask_obs with header seq: 22044 +stamp: + secs: 1733935278 + nsecs: 607197761 +frame_id: '' and grasped_object_mask header seq: 508872 +stamp: + secs: 1733935278 + nsecs: 370348692 +frame_id: "camera_color_optical_frame" +[INFO] [1733935278.861007]: received mask_obs with header seq: 22045 +stamp: + secs: 1733935278 + nsecs: 865187644 +frame_id: '' and grasped_object_mask header seq: 508880 +stamp: + secs: 1733935278 + nsecs: 636795282 +frame_id: "camera_color_optical_frame" +[INFO] [1733935279.098058]: received mask_obs with header seq: 22046 +stamp: + secs: 1733935279 + nsecs: 102196931 +frame_id: '' and grasped_object_mask header seq: 508887 +stamp: + secs: 1733935278 + nsecs: 869935989 +frame_id: "camera_color_optical_frame" +[INFO] [1733935279.337583]: received mask_obs with header seq: 22047 +stamp: + secs: 1733935279 + nsecs: 341685771 +frame_id: '' and grasped_object_mask header seq: 508894 +stamp: + secs: 1733935279 + nsecs: 103076696 +frame_id: "camera_color_optical_frame" +[INFO] [1733935279.586029]: received mask_obs with header seq: 22048 +stamp: + secs: 1733935279 + nsecs: 590155363 +frame_id: '' and grasped_object_mask header seq: 508901 +stamp: + secs: 1733935279 + nsecs: 336215496 +frame_id: "camera_color_optical_frame" +[INFO] [1733935279.840936]: received mask_obs with header seq: 22049 +stamp: + secs: 1733935279 + nsecs: 845028400 +frame_id: '' and grasped_object_mask header seq: 508909 +stamp: + secs: 1733935279 + nsecs: 602661371 +frame_id: "camera_color_optical_frame" +[INFO] [1733935280.099014]: received mask_obs with header seq: 22050 +stamp: + secs: 1733935280 + nsecs: 103171110 +frame_id: '' and grasped_object_mask header seq: 508917 +stamp: + secs: 1733935279 + nsecs: 869105577 +frame_id: "camera_color_optical_frame" +[INFO] [1733935280.338437]: received mask_obs with header seq: 22051 +stamp: + secs: 1733935280 + nsecs: 342628240 +frame_id: '' and grasped_object_mask header seq: 508924 +stamp: + secs: 1733935280 + nsecs: 102242708 +frame_id: "camera_color_optical_frame" +[INFO] [1733935280.599646]: received mask_obs with header seq: 22052 +stamp: + secs: 1733935280 + nsecs: 603679656 +frame_id: '' and grasped_object_mask header seq: 508932 +stamp: + secs: 1733935280 + nsecs: 368685484 +frame_id: "camera_color_optical_frame" +[INFO] [1733935280.836004]: received mask_obs with header seq: 22053 +stamp: + secs: 1733935280 + nsecs: 840143680 +frame_id: '' and grasped_object_mask header seq: 508939 +stamp: + secs: 1733935280 + nsecs: 601821184 +frame_id: "camera_color_optical_frame" +[INFO] [1733935281.078340]: received mask_obs with header seq: 22054 +stamp: + secs: 1733935281 + nsecs: 82428693 +frame_id: '' and grasped_object_mask header seq: 508946 +stamp: + secs: 1733935280 + nsecs: 834957361 +frame_id: "camera_color_optical_frame" +[INFO] [1733935281.320443]: received mask_obs with header seq: 22055 +stamp: + secs: 1733935281 + nsecs: 324586153 +frame_id: '' and grasped_object_mask header seq: 508954 +stamp: + secs: 1733935281 + nsecs: 101403475 +frame_id: "camera_color_optical_frame" +[INFO] [1733935281.575445]: received mask_obs with header seq: 22056 +stamp: + secs: 1733935281 + nsecs: 579533815 +frame_id: '' and grasped_object_mask header seq: 508961 +stamp: + secs: 1733935281 + nsecs: 334551334 +frame_id: "camera_color_optical_frame" +[INFO] [1733935281.823393]: received mask_obs with header seq: 22057 +stamp: + secs: 1733935281 + nsecs: 827114820 +frame_id: '' and grasped_object_mask header seq: 508969 +stamp: + secs: 1733935281 + nsecs: 601005316 +frame_id: "camera_color_optical_frame" +[INFO] [1733935282.081633]: received mask_obs with header seq: 22058 +stamp: + secs: 1733935282 + nsecs: 85840463 +frame_id: '' and grasped_object_mask header seq: 508976 +stamp: + secs: 1733935281 + nsecs: 834153652 +frame_id: "camera_color_optical_frame" +[INFO] [1733935282.342731]: received mask_obs with header seq: 22059 +stamp: + secs: 1733935282 + nsecs: 346517801 +frame_id: '' and grasped_object_mask header seq: 508984 +stamp: + secs: 1733935282 + nsecs: 100607634 +frame_id: "camera_color_optical_frame" +[INFO] [1733935282.589285]: received mask_obs with header seq: 22060 +stamp: + secs: 1733935282 + nsecs: 592802762 +frame_id: '' and grasped_object_mask header seq: 508992 +stamp: + secs: 1733935282 + nsecs: 367062092 +frame_id: "camera_color_optical_frame" +[INFO] [1733935282.844315]: received mask_obs with header seq: 22061 +stamp: + secs: 1733935282 + nsecs: 848241090 +frame_id: '' and grasped_object_mask header seq: 508999 +stamp: + secs: 1733935282 + nsecs: 600210190 +frame_id: "camera_color_optical_frame" +[INFO] [1733935283.104441]: received mask_obs with header seq: 22062 +stamp: + secs: 1733935283 + nsecs: 108578920 +frame_id: '' and grasped_object_mask header seq: 509007 +stamp: + secs: 1733935282 + nsecs: 866664886 +frame_id: "camera_color_optical_frame" +[INFO] [1733935283.360662]: received mask_obs with header seq: 22063 +stamp: + secs: 1733935283 + nsecs: 364868402 +frame_id: '' and grasped_object_mask header seq: 509015 +stamp: + secs: 1733935283 + nsecs: 133111238 +frame_id: "camera_color_optical_frame" +[INFO] [1733935283.603353]: received mask_obs with header seq: 22064 +stamp: + secs: 1733935283 + nsecs: 607386350 +frame_id: '' and grasped_object_mask header seq: 509022 +stamp: + secs: 1733935283 + nsecs: 366246223 +frame_id: "camera_color_optical_frame" +[INFO] [1733935283.860399]: received mask_obs with header seq: 22065 +stamp: + secs: 1733935283 + nsecs: 864587068 +frame_id: '' and grasped_object_mask header seq: 509030 +stamp: + secs: 1733935283 + nsecs: 632685661 +frame_id: "camera_color_optical_frame" +[INFO] [1733935284.104061]: received mask_obs with header seq: 22066 +stamp: + secs: 1733935284 + nsecs: 107969760 +frame_id: '' and grasped_object_mask header seq: 509037 +stamp: + secs: 1733935283 + nsecs: 865819693 +frame_id: "camera_color_optical_frame" +[INFO] [1733935284.360459]: received mask_obs with header seq: 22067 +stamp: + secs: 1733935284 + nsecs: 364593267 +frame_id: '' and grasped_object_mask header seq: 509045 +stamp: + secs: 1733935284 + nsecs: 132267475 +frame_id: "camera_color_optical_frame" +[INFO] [1733935284.598140]: received mask_obs with header seq: 22068 +stamp: + secs: 1733935284 + nsecs: 602252960 +frame_id: '' and grasped_object_mask header seq: 509052 +stamp: + secs: 1733935284 + nsecs: 365416765 +frame_id: "camera_color_optical_frame" +[INFO] [1733935284.838303]: received mask_obs with header seq: 22069 +stamp: + secs: 1733935284 + nsecs: 842395067 +frame_id: '' and grasped_object_mask header seq: 509059 +stamp: + secs: 1733935284 + nsecs: 598566055 +frame_id: "camera_color_optical_frame" +[INFO] [1733935285.103577]: received mask_obs with header seq: 22070 +stamp: + secs: 1733935285 + nsecs: 107409954 +frame_id: '' and grasped_object_mask header seq: 509067 +stamp: + secs: 1733935284 + nsecs: 865022182 +frame_id: "camera_color_optical_frame" +[INFO] [1733935285.371047]: received mask_obs with header seq: 22071 +stamp: + secs: 1733935285 + nsecs: 375306844 +frame_id: '' and grasped_object_mask header seq: 509075 +stamp: + secs: 1733935285 + nsecs: 131474495 +frame_id: "camera_color_optical_frame" +[INFO] [1733935285.619250]: received mask_obs with header seq: 22072 +stamp: + secs: 1733935285 + nsecs: 623445272 +frame_id: '' and grasped_object_mask header seq: 509082 +stamp: + secs: 1733935285 + nsecs: 364618063 +frame_id: "camera_color_optical_frame" +[INFO] [1733935285.873566]: received mask_obs with header seq: 22073 +stamp: + secs: 1733935285 + nsecs: 877703905 +frame_id: '' and grasped_object_mask header seq: 509090 +stamp: + secs: 1733935285 + nsecs: 631067514 +frame_id: "camera_color_optical_frame" +[INFO] [1733935286.130674]: received mask_obs with header seq: 22074 +stamp: + secs: 1733935286 + nsecs: 134832143 +frame_id: '' and grasped_object_mask header seq: 509098 +stamp: + secs: 1733935285 + nsecs: 897516489 +frame_id: "camera_color_optical_frame" +[INFO] [1733935286.389396]: received mask_obs with header seq: 22075 +stamp: + secs: 1733935286 + nsecs: 393568038 +frame_id: '' and grasped_object_mask header seq: 509106 +stamp: + secs: 1733935286 + nsecs: 163964748 +frame_id: "camera_color_optical_frame" +[INFO] [1733935286.625067]: received mask_obs with header seq: 22076 +stamp: + secs: 1733935286 + nsecs: 629197359 +frame_id: '' and grasped_object_mask header seq: 509113 +stamp: + secs: 1733935286 + nsecs: 397106886 +frame_id: "camera_color_optical_frame" +[INFO] [1733935286.863715]: received mask_obs with header seq: 22077 +stamp: + secs: 1733935286 + nsecs: 867812395 +frame_id: '' and grasped_object_mask header seq: 509120 +stamp: + secs: 1733935286 + nsecs: 630248785 +frame_id: "camera_color_optical_frame" +[INFO] [1733935287.105382]: received mask_obs with header seq: 22078 +stamp: + secs: 1733935287 + nsecs: 109485864 +frame_id: '' and grasped_object_mask header seq: 509127 +stamp: + secs: 1733935286 + nsecs: 863389969 +frame_id: "camera_color_optical_frame" +[INFO] [1733935287.346066]: received mask_obs with header seq: 22079 +stamp: + secs: 1733935287 + nsecs: 350147247 +frame_id: '' and grasped_object_mask header seq: 509134 +stamp: + secs: 1733935287 + nsecs: 96535683 +frame_id: "camera_color_optical_frame" +[INFO] [1733935287.600868]: received mask_obs with header seq: 22080 +stamp: + secs: 1733935287 + nsecs: 604967594 +frame_id: '' and grasped_object_mask header seq: 509142 +stamp: + secs: 1733935287 + nsecs: 362992048 +frame_id: "camera_color_optical_frame" +[INFO] [1733935287.837203]: received mask_obs with header seq: 22081 +stamp: + secs: 1733935287 + nsecs: 841414451 +frame_id: '' and grasped_object_mask header seq: 509149 +stamp: + secs: 1733935287 + nsecs: 596142054 +frame_id: "camera_color_optical_frame" +[INFO] [1733935288.091324]: received mask_obs with header seq: 22082 +stamp: + secs: 1733935288 + nsecs: 95262289 +frame_id: '' and grasped_object_mask header seq: 509156 +stamp: + secs: 1733935287 + nsecs: 829292774 +frame_id: "camera_color_optical_frame" +[INFO] [1733935288.334097]: received mask_obs with header seq: 22083 +stamp: + secs: 1733935288 + nsecs: 337235689 +frame_id: '' and grasped_object_mask header seq: 509164 +stamp: + secs: 1733935288 + nsecs: 95751047 +frame_id: "camera_color_optical_frame" +[INFO] [1733935288.577502]: received mask_obs with header seq: 22084 +stamp: + secs: 1733935288 + nsecs: 581673383 +frame_id: '' and grasped_object_mask header seq: 509171 +stamp: + secs: 1733935288 + nsecs: 328904390 +frame_id: "camera_color_optical_frame" +[INFO] [1733935288.805827]: received mask_obs with header seq: 22085 +stamp: + secs: 1733935288 + nsecs: 809909105 +frame_id: '' and grasped_object_mask header seq: 509178 +stamp: + secs: 1733935288 + nsecs: 562056541 +frame_id: "camera_color_optical_frame" +[INFO] [1733935289.055206]: received mask_obs with header seq: 22086 +stamp: + secs: 1733935289 + nsecs: 57225227 +frame_id: '' and grasped_object_mask header seq: 509186 +stamp: + secs: 1733935288 + nsecs: 828518867 +frame_id: "camera_color_optical_frame" +[INFO] [1733935289.304369]: received mask_obs with header seq: 22087 +stamp: + secs: 1733935289 + nsecs: 308372735 +frame_id: '' and grasped_object_mask header seq: 509193 +stamp: + secs: 1733935289 + nsecs: 61670303 +frame_id: "camera_color_optical_frame" +[INFO] [1733935289.571916]: received mask_obs with header seq: 22088 +stamp: + secs: 1733935289 + nsecs: 576134920 +frame_id: '' and grasped_object_mask header seq: 509201 +stamp: + secs: 1733935289 + nsecs: 328126669 +frame_id: "camera_color_optical_frame" +[INFO] [1733935289.821467]: received mask_obs with header seq: 22089 +stamp: + secs: 1733935289 + nsecs: 824393510 +frame_id: '' and grasped_object_mask header seq: 509209 +stamp: + secs: 1733935289 + nsecs: 594584465 +frame_id: "camera_color_optical_frame" +[INFO] [1733935290.073719]: received mask_obs with header seq: 22090 +stamp: + secs: 1733935290 + nsecs: 77884674 +frame_id: '' and grasped_object_mask header seq: 509216 +stamp: + secs: 1733935289 + nsecs: 827733755 +frame_id: "camera_color_optical_frame" +[INFO] [1733935290.320356]: received mask_obs with header seq: 22091 +stamp: + secs: 1733935290 + nsecs: 323549270 +frame_id: '' and grasped_object_mask header seq: 509224 +stamp: + secs: 1733935290 + nsecs: 94188929 +frame_id: "camera_color_optical_frame" +[INFO] [1733935290.570356]: received mask_obs with header seq: 22092 +stamp: + secs: 1733935290 + nsecs: 574417591 +frame_id: '' and grasped_object_mask header seq: 509231 +stamp: + secs: 1733935290 + nsecs: 327332973 +frame_id: "camera_color_optical_frame" +[INFO] [1733935290.826464]: received mask_obs with header seq: 22093 +stamp: + secs: 1733935290 + nsecs: 829451084 +frame_id: '' and grasped_object_mask header seq: 509238 +stamp: + secs: 1733935290 + nsecs: 560477972 +frame_id: "camera_color_optical_frame" +[INFO] [1733935291.069844]: received mask_obs with header seq: 22094 +stamp: + secs: 1733935291 + nsecs: 73856830 +frame_id: '' and grasped_object_mask header seq: 509246 +stamp: + secs: 1733935290 + nsecs: 826928854 +frame_id: "camera_color_optical_frame" +[INFO] [1733935291.325150]: received mask_obs with header seq: 22095 +stamp: + secs: 1733935291 + nsecs: 329347372 +frame_id: '' and grasped_object_mask header seq: 509254 +stamp: + secs: 1733935291 + nsecs: 93375683 +frame_id: "camera_color_optical_frame" +[INFO] [1733935291.580230]: received mask_obs with header seq: 22096 +stamp: + secs: 1733935291 + nsecs: 584431409 +frame_id: '' and grasped_object_mask header seq: 509261 +stamp: + secs: 1733935291 + nsecs: 326511621 +frame_id: "camera_color_optical_frame" +[INFO] [1733935291.821985]: received mask_obs with header seq: 22097 +stamp: + secs: 1733935291 + nsecs: 824577569 +frame_id: '' and grasped_object_mask header seq: 509269 +stamp: + secs: 1733935291 + nsecs: 592952013 +frame_id: "camera_color_optical_frame" +[INFO] [1733935292.082634]: received mask_obs with header seq: 22098 +stamp: + secs: 1733935292 + nsecs: 86860656 +frame_id: '' and grasped_object_mask header seq: 509277 +stamp: + secs: 1733935291 + nsecs: 859391451 +frame_id: "camera_color_optical_frame" +[INFO] [1733935292.324540]: received mask_obs with header seq: 22099 +stamp: + secs: 1733935292 + nsecs: 328421831 +frame_id: '' and grasped_object_mask header seq: 509284 +stamp: + secs: 1733935292 + nsecs: 92530012 +frame_id: "camera_color_optical_frame" +[INFO] [1733935292.589087]: received mask_obs with header seq: 22100 +stamp: + secs: 1733935292 + nsecs: 593283891 +frame_id: '' and grasped_object_mask header seq: 509292 +stamp: + secs: 1733935292 + nsecs: 358978033 +frame_id: "camera_color_optical_frame" +[INFO] [1733935292.841901]: received mask_obs with header seq: 22101 +stamp: + secs: 1733935292 + nsecs: 845422029 +frame_id: '' and grasped_object_mask header seq: 509300 +stamp: + secs: 1733935292 + nsecs: 625426769 +frame_id: "camera_color_optical_frame" +[INFO] [1733935293.083419]: received mask_obs with header seq: 22102 +stamp: + secs: 1733935293 + nsecs: 87143421 +frame_id: '' and grasped_object_mask header seq: 509307 +stamp: + secs: 1733935292 + nsecs: 858567953 +frame_id: "camera_color_optical_frame" +[INFO] [1733935293.321194]: received mask_obs with header seq: 22103 +stamp: + secs: 1733935293 + nsecs: 325015068 +frame_id: '' and grasped_object_mask header seq: 509314 +stamp: + secs: 1733935293 + nsecs: 91714144 +frame_id: "camera_color_optical_frame" +[INFO] [1733935293.570291]: received mask_obs with header seq: 22104 +stamp: + secs: 1733935293 + nsecs: 574228525 +frame_id: '' and grasped_object_mask header seq: 509322 +stamp: + secs: 1733935293 + nsecs: 358171940 +frame_id: "camera_color_optical_frame" +[INFO] [1733935293.825076]: received mask_obs with header seq: 22105 +stamp: + secs: 1733935293 + nsecs: 829060316 +frame_id: '' and grasped_object_mask header seq: 509329 +stamp: + secs: 1733935293 + nsecs: 591323614 +frame_id: "camera_color_optical_frame" +[INFO] [1733935294.067469]: received mask_obs with header seq: 22106 +stamp: + secs: 1733935294 + nsecs: 71625947 +frame_id: '' and grasped_object_mask header seq: 509336 +stamp: + secs: 1733935293 + nsecs: 824475527 +frame_id: "camera_color_optical_frame" +[INFO] [1733935294.340741]: received mask_obs with header seq: 22107 +stamp: + secs: 1733935294 + nsecs: 344866514 +frame_id: '' and grasped_object_mask header seq: 509344 +stamp: + secs: 1733935294 + nsecs: 90931892 +frame_id: "camera_color_optical_frame" +[INFO] [1733935294.602688]: received mask_obs with header seq: 22108 +stamp: + secs: 1733935294 + nsecs: 606296777 +frame_id: '' and grasped_object_mask header seq: 509353 +stamp: + secs: 1733935294 + nsecs: 390690565 +frame_id: "camera_color_optical_frame" +[INFO] [1733935294.858014]: received mask_obs with header seq: 22109 +stamp: + secs: 1733935294 + nsecs: 861686229 +frame_id: '' and grasped_object_mask header seq: 509360 +stamp: + secs: 1733935294 + nsecs: 623835802 +frame_id: "camera_color_optical_frame" +[INFO] [1733935295.101590]: received mask_obs with header seq: 22110 +stamp: + secs: 1733935295 + nsecs: 105779409 +frame_id: '' and grasped_object_mask header seq: 509367 +stamp: + secs: 1733935294 + nsecs: 856981754 +frame_id: "camera_color_optical_frame" +[INFO] [1733935295.364056]: received mask_obs with header seq: 22111 +stamp: + secs: 1733935295 + nsecs: 368102073 +frame_id: '' and grasped_object_mask header seq: 509375 +stamp: + secs: 1733935295 + nsecs: 123432159 +frame_id: "camera_color_optical_frame" +[INFO] [1733935295.627070]: received mask_obs with header seq: 22112 +stamp: + secs: 1733935295 + nsecs: 631273031 +frame_id: '' and grasped_object_mask header seq: 509383 +stamp: + secs: 1733935295 + nsecs: 389881372 +frame_id: "camera_color_optical_frame" +[INFO] [1733935295.869143]: received mask_obs with header seq: 22113 +stamp: + secs: 1733935295 + nsecs: 873281002 +frame_id: '' and grasped_object_mask header seq: 509391 +stamp: + secs: 1733935295 + nsecs: 656331301 +frame_id: "camera_color_optical_frame" +[INFO] [1733935296.122864]: received mask_obs with header seq: 22114 +stamp: + secs: 1733935296 + nsecs: 126703977 +frame_id: '' and grasped_object_mask header seq: 509398 +stamp: + secs: 1733935295 + nsecs: 889473677 +frame_id: "camera_color_optical_frame" +[INFO] [1733935296.364064]: received mask_obs with header seq: 22115 +stamp: + secs: 1733935296 + nsecs: 368048429 +frame_id: '' and grasped_object_mask header seq: 509405 +stamp: + secs: 1733935296 + nsecs: 122617960 +frame_id: "camera_color_optical_frame" +[INFO] [1733935296.622860]: received mask_obs with header seq: 22116 +stamp: + secs: 1733935296 + nsecs: 626951456 +frame_id: '' and grasped_object_mask header seq: 509413 +stamp: + secs: 1733935296 + nsecs: 389068604 +frame_id: "camera_color_optical_frame" +[INFO] [1733935296.879571]: received mask_obs with header seq: 22117 +stamp: + secs: 1733935296 + nsecs: 882599830 +frame_id: '' and grasped_object_mask header seq: 509421 +stamp: + secs: 1733935296 + nsecs: 655519247 +frame_id: "camera_color_optical_frame" +[INFO] [1733935297.118312]: received mask_obs with header seq: 22118 +stamp: + secs: 1733935297 + nsecs: 122487068 +frame_id: '' and grasped_object_mask header seq: 509428 +stamp: + secs: 1733935296 + nsecs: 888664007 +frame_id: "camera_color_optical_frame" +[INFO] [1733935297.381134]: received mask_obs with header seq: 22119 +stamp: + secs: 1733935297 + nsecs: 385061740 +frame_id: '' and grasped_object_mask header seq: 509436 +stamp: + secs: 1733935297 + nsecs: 155123234 +frame_id: "camera_color_optical_frame" +[INFO] [1733935297.640433]: received mask_obs with header seq: 22120 +stamp: + secs: 1733935297 + nsecs: 644598245 +frame_id: '' and grasped_object_mask header seq: 509444 +stamp: + secs: 1733935297 + nsecs: 421587467 +frame_id: "camera_color_optical_frame" +[INFO] [1733935297.886146]: received mask_obs with header seq: 22121 +stamp: + secs: 1733935297 + nsecs: 890255451 +frame_id: '' and grasped_object_mask header seq: 509451 +stamp: + secs: 1733935297 + nsecs: 654744625 +frame_id: "camera_color_optical_frame" +[INFO] [1733935298.134082]: received mask_obs with header seq: 22122 +stamp: + secs: 1733935298 + nsecs: 138226509 +frame_id: '' and grasped_object_mask header seq: 509458 +stamp: + secs: 1733935297 + nsecs: 887902260 +frame_id: "camera_color_optical_frame" +[INFO] [1733935298.385826]: received mask_obs with header seq: 22123 +stamp: + secs: 1733935298 + nsecs: 390051603 +frame_id: '' and grasped_object_mask header seq: 509466 +stamp: + secs: 1733935298 + nsecs: 154362679 +frame_id: "camera_color_optical_frame" +[INFO] [1733935298.634900]: received mask_obs with header seq: 22124 +stamp: + secs: 1733935298 + nsecs: 638223648 +frame_id: '' and grasped_object_mask header seq: 509473 +stamp: + secs: 1733935298 + nsecs: 387514114 +frame_id: "camera_color_optical_frame" +[INFO] [1733935298.888506]: received mask_obs with header seq: 22125 +stamp: + secs: 1733935298 + nsecs: 892633199 +frame_id: '' and grasped_object_mask header seq: 509481 +stamp: + secs: 1733935298 + nsecs: 653972626 +frame_id: "camera_color_optical_frame" +[INFO] [1733935299.139613]: received mask_obs with header seq: 22126 +stamp: + secs: 1733935299 + nsecs: 142855882 +frame_id: '' and grasped_object_mask header seq: 509489 +stamp: + secs: 1733935298 + nsecs: 920431376 +frame_id: "camera_color_optical_frame" +[INFO] [1733935299.402965]: received mask_obs with header seq: 22127 +stamp: + secs: 1733935299 + nsecs: 405007600 +frame_id: '' and grasped_object_mask header seq: 509497 +stamp: + secs: 1733935299 + nsecs: 186876297 +frame_id: "camera_color_optical_frame" +[INFO] [1733935299.658538]: received mask_obs with header seq: 22128 +stamp: + secs: 1733935299 + nsecs: 662704706 +frame_id: '' and grasped_object_mask header seq: 509504 +stamp: + secs: 1733935299 + nsecs: 420011997 +frame_id: "camera_color_optical_frame" +[INFO] [1733935299.770546]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit[pub_text-13] killing on exit +[INFO] [1733935299.921619]: received mask_obs with header seq: 22129 +stamp: + secs: 1733935299 + nsecs: 924869060 +frame_id: '' and grasped_object_mask header seq: 509512 +stamp: + secs: 1733935299 + nsecs: 686451912 +frame_id: "camera_color_optical_frame" +[INFO] [1733935300.548376]: Stored episode 5. +[INFO] [1733935300.553457]: Saving video... +[INFO] [1733935300.772136]: Video saved! +[INFO] [1733935300.813064]: resetting environment +[INFO] [1733935300.858858]: cleared current plan +[INFO] [1733935300.866116]: moving to home +[INFO] [1733935303.675164]: reached home +[INFO] [1733935303.686254]: reset action history +[INFO] [1733935306.499879]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! \ No newline at end of file diff --git a/113638/wandb/run-20241211_113644-ahosb0lo/files/requirements.txt b/113638/wandb/run-20241211_113644-ahosb0lo/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..192179a195956e252bda33f2342b4970a9de79d9 --- /dev/null +++ b/113638/wandb/run-20241211_113644-ahosb0lo/files/requirements.txt @@ -0,0 +1,339 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.2 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-metadata.json b/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..4557272cfc44e383808aa0ed6d277fa6b1c5a39d --- /dev/null +++ b/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2024-12-11T16:36:45.012114", + "startedAt": "2024-12-11T16:36:44.613038", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "1b557f164c7f62a8d1545c761b13ede1677c3a83" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3937.48675, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.999, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.011, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 661.6516265869141 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71596908569336 + } +} diff --git a/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json b/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..dc0e0a112e388c2eb573c13c66bd96f80da85dae --- /dev/null +++ b/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "dea3cb7ce16ad057bd5fcb35b50592a86feeb9f7d6f3bd7b6f42095d27997931", "size": 1422243, "path": "media/videos/eval/0_eval_0_dea3cb7ce16ad057bd5f.mp4"}, "global_step": 680, "_timestamp": 1733935311.290467, "_runtime": 306.669842004776, "_step": 9, "eval/1_eval": {"_type": "video-file", "sha256": "925210ff3b36d88f3887c6bf561b993eeb2172f9b898edf7efaa91f7c720e590", "size": 1468875, "path": "media/videos/eval/1_eval_1_925210ff3b36d88f3887.mp4"}, "eval/num_success": 4.0, "episode": 5.0, "eval/success_rate": 0.800000011920929, "eval/2_eval": {"_type": "video-file", "sha256": "ddba0e3583786ac1097bd8d918d5801eb4867cac0e908be1695101412c576aec", "size": 1498755, "path": "media/videos/eval/2_eval_2_ddba0e3583786ac1097b.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "b687720b2d32aeeb9a91a18ae8924160e7860fcc22b2850d7f2b3b59ca27c2ec", "size": 1454836, "path": "media/videos/eval/3_eval_3_b687720b2d32aeeb9a91.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "10441113b4d3ea44e8c6df553d629edb8882f588fa5814fc108ba5370ac0ec0d", "size": 1451627, "path": "media/videos/eval/4_eval_4_10441113b4d3ea44e8c6.mp4"}, "_wandb": {"runtime": 306}} \ No newline at end of file diff --git a/113638/wandb/run-20241211_113644-ahosb0lo/logs/debug-internal.log b/113638/wandb/run-20241211_113644-ahosb0lo/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c56fb179ccad396ea87cb0c61ff4691bc6f0f4d3 --- /dev/null +++ b/113638/wandb/run-20241211_113644-ahosb0lo/logs/debug-internal.log @@ -0,0 +1,840 @@ +2024-12-11 11:36:44,620 INFO StreamThr :743447 [internal.py:wandb_internal():85] W&B internal server running at pid: 743447, started at: 2024-12-11 11:36:44.620258 +2024-12-11 11:36:44,621 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status +2024-12-11 11:36:44,632 INFO WriterThread:743447 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/run-ahosb0lo.wandb +2024-12-11 11:36:44,633 DEBUG SenderThread:743447 [sender.py:send():379] send: header +2024-12-11 11:36:44,642 DEBUG SenderThread:743447 [sender.py:send():379] send: run +2024-12-11 11:36:44,901 INFO SenderThread:743447 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files +2024-12-11 11:36:44,901 INFO SenderThread:743447 [sender.py:_start_run_threads():1188] run started: ahosb0lo with start time 1733935004.620625 +2024-12-11 11:36:44,921 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: check_version +2024-12-11 11:36:44,921 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: check_version +2024-12-11 11:36:44,995 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: run_start +2024-12-11 11:36:45,000 DEBUG HandlerThread:743447 [system_info.py:__init__():26] System info init +2024-12-11 11:36:45,000 DEBUG HandlerThread:743447 [system_info.py:__init__():41] System info init done +2024-12-11 11:36:45,000 INFO HandlerThread:743447 [system_monitor.py:start():194] Starting system monitor +2024-12-11 11:36:45,000 INFO SystemMonitor:743447 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-11 11:36:45,000 INFO HandlerThread:743447 [system_monitor.py:probe():214] Collecting system info +2024-12-11 11:36:45,000 INFO SystemMonitor:743447 [interfaces.py:start():188] Started cpu monitoring +2024-12-11 11:36:45,001 INFO SystemMonitor:743447 [interfaces.py:start():188] Started disk monitoring +2024-12-11 11:36:45,002 INFO SystemMonitor:743447 [interfaces.py:start():188] Started gpu monitoring +2024-12-11 11:36:45,002 INFO SystemMonitor:743447 [interfaces.py:start():188] Started memory monitoring +2024-12-11 11:36:45,003 INFO SystemMonitor:743447 [interfaces.py:start():188] Started network monitoring +2024-12-11 11:36:45,012 DEBUG HandlerThread:743447 [system_info.py:probe():152] Probing system +2024-12-11 11:36:45,013 DEBUG HandlerThread:743447 [system_info.py:_probe_git():137] Probing git +2024-12-11 11:36:45,018 DEBUG HandlerThread:743447 [system_info.py:_probe_git():145] Probing git done +2024-12-11 11:36:45,018 DEBUG HandlerThread:743447 [system_info.py:probe():200] Probing system done +2024-12-11 11:36:45,018 DEBUG HandlerThread:743447 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-11T16:36:45.012114', 'startedAt': '2024-12-11T16:36:44.613038', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '1b557f164c7f62a8d1545c761b13ede1677c3a83'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3937.48675, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.011, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 661.6516265869141}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71596908569336}} +2024-12-11 11:36:45,018 INFO HandlerThread:743447 [system_monitor.py:probe():224] Finished collecting system info +2024-12-11 11:36:45,018 INFO HandlerThread:743447 [system_monitor.py:probe():227] Publishing system info +2024-12-11 11:36:45,018 DEBUG HandlerThread:743447 [system_info.py:_save_code():44] Saving code +2024-12-11 11:36:45,020 DEBUG HandlerThread:743447 [system_info.py:_save_code():67] Saving code done +2024-12-11 11:36:45,020 DEBUG HandlerThread:743447 [system_info.py:_save_patches():84] Saving git patches +2024-12-11 11:36:45,082 DEBUG HandlerThread:743447 [system_info.py:_save_patches():126] Saving git patches done +2024-12-11 11:36:45,082 INFO HandlerThread:743447 [system_monitor.py:probe():229] Finished publishing system info +2024-12-11 11:36:45,086 DEBUG SenderThread:743447 [sender.py:send():379] send: files +2024-12-11 11:36:45,086 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-11 11:36:45,086 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-11 11:36:45,086 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-11 11:36:45,328 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: python_packages +2024-12-11 11:36:45,328 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:36:45,329 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: python_packages +2024-12-11 11:36:45,329 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:36:45,330 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:45,331 INFO HandlerThread:743447 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/tb" + save: true +} +control { + relay_id: "3b814f0f1e304546b568a989da857bea" +} +_info { + stream_id: "ahosb0lo" +} + +2024-12-11 11:36:45,331 DEBUG HandlerThread:743447 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-11 11:36:45,344 INFO wandb-upload_0:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/jpxgwjga-wandb-metadata.json +2024-12-11 11:36:45,381 INFO wandb-upload_1:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/xzmf7vax-code/FISH/eval_robot.py +2024-12-11 11:36:45,383 INFO wandb-upload_2:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/am9eq8ny-diff.patch +2024-12-11 11:36:45,439 DEBUG SenderThread:743447 [sender.py:send():379] send: telemetry +2024-12-11 11:36:45,439 DEBUG SenderThread:743447 [sender.py:send():379] send: tbrecord +2024-12-11 11:36:45,439 DEBUG SenderThread:743447 [sender.py:send():379] send: files +2024-12-11 11:36:45,439 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 with policy live +2024-12-11 11:36:45,614 INFO wandb-upload_0:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/sdv9591m-events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:36:45,902 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-metadata.json +2024-12-11 11:36:45,902 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/diff.patch +2024-12-11 11:36:45,903 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:36:45,903 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/requirements.txt +2024-12-11 11:36:45,903 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:36:45,903 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/code/FISH/eval_robot.py +2024-12-11 11:36:45,903 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/code/FISH +2024-12-11 11:36:45,903 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/code +2024-12-11 11:36:46,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:47,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:47,902 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:36:48,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:49,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:49,565 DEBUG SenderThread:743447 [sender.py:send():379] send: config +2024-12-11 11:36:50,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:36:50,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:51,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:51,903 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:36:52,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:53,330 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:53,903 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:36:54,346 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:55,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:36:55,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:55,904 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:36:56,330 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:57,330 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:57,905 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:36:58,330 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:36:59,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:00,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:37:00,329 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:37:00,370 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:00,483 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:01,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:02,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:03,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:04,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:05,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:06,330 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:06,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:07,332 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:08,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:09,332 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:10,332 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:11,330 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:11,332 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:12,332 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:13,332 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:14,332 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:15,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:37:15,329 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:37:15,370 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:16,333 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:16,419 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:16,910 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/config.yaml +2024-12-11 11:37:17,333 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:18,333 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:19,333 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:19,911 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:20,333 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:21,333 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:21,687 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:22,334 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:23,334 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:23,912 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:24,334 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:25,334 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:25,912 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:26,334 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:26,799 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:27,336 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:27,913 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:28,336 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:29,337 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:29,913 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:30,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:37:30,329 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:37:30,337 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:31,337 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:31,803 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:31,914 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:32,337 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:33,338 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:33,914 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:34,337 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:35,338 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:35,915 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:36,338 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:37,047 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:37,338 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:37,916 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:38,339 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:39,339 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:39,916 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:40,339 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:41,339 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:41,917 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:42,166 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:42,339 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:43,340 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:43,918 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:44,340 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:45,003 DEBUG SystemMonitor:743447 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-11 11:37:45,004 DEBUG SenderThread:743447 [sender.py:send():379] send: stats +2024-12-11 11:37:45,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:37:45,329 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:37:45,340 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:45,918 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:46,340 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:47,340 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:47,422 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:47,919 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:48,340 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:49,340 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:49,919 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:50,340 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:51,341 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:51,920 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:52,341 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:52,603 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:53,341 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:53,921 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:54,341 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:55,341 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:55,921 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:56,341 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:57,288 DEBUG SenderThread:743447 [sender.py:send():379] send: files +2024-12-11 11:37:57,289 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:37:57,289 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_dea3cb7ce16ad057bd5f.mp4 with policy now +2024-12-11 11:37:57,290 DEBUG SenderThread:743447 [sender.py:send():379] send: history +2024-12-11 11:37:57,290 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:37:57,290 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:37:57,342 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:57,529 INFO wandb-upload_2:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/s4xw9bmh-media/videos/eval/0_eval_0_dea3cb7ce16ad057bd5f.mp4 +2024-12-11 11:37:57,922 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:37:57,922 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/0_eval_0_dea3cb7ce16ad057bd5f.mp4 +2024-12-11 11:37:57,923 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json +2024-12-11 11:37:57,923 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos +2024-12-11 11:37:57,923 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval +2024-12-11 11:37:57,923 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media +2024-12-11 11:37:58,202 INFO wandb-upload_0:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/7xrrwz3m-media/videos/eval/0_eval_0_dea3cb7ce16ad057bd5f.mp4 +2024-12-11 11:37:58,302 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:37:58,342 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:59,342 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:37:59,923 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:00,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:38:00,329 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:38:00,342 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:01,342 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:01,923 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:02,342 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:03,345 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:03,924 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:03,962 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:38:04,344 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:05,344 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:06,344 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:07,344 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:07,925 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:38:08,171 INFO wandb-upload_2:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/pvc3l7fp-events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:38:08,345 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:09,345 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:09,794 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:38:09,926 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:10,345 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:11,346 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:12,345 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:13,346 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:13,927 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:14,348 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:14,851 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:38:15,004 DEBUG SenderThread:743447 [sender.py:send():379] send: stats +2024-12-11 11:38:15,329 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:38:15,330 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:38:15,346 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:15,928 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:16,346 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:17,347 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:17,929 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:18,347 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:19,347 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:19,929 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:20,073 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:38:20,347 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:21,348 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:21,930 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:22,347 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:23,347 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:23,931 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:24,347 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:25,115 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:38:25,347 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:25,931 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:26,347 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:27,348 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:27,932 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:28,348 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:29,348 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:29,933 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:30,147 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:38:30,330 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:38:30,330 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:38:30,349 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:31,348 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:31,934 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:32,349 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:33,348 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:33,934 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:34,348 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:35,171 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:38:35,349 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:35,935 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:36,350 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:37,350 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:37,936 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:38,350 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:39,350 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:39,936 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:40,215 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:38:40,352 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:41,353 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:41,937 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:42,354 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:43,354 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:43,938 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:44,354 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:45,004 DEBUG SenderThread:743447 [sender.py:send():379] send: stats +2024-12-11 11:38:45,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:38:45,331 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:38:45,354 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:45,447 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:38:45,938 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:46,354 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:47,357 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:47,749 DEBUG SenderThread:743447 [sender.py:send():379] send: files +2024-12-11 11:38:47,749 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_925210ff3b36d88f3887.mp4 with policy now +2024-12-11 11:38:47,750 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:38:47,751 DEBUG SenderThread:743447 [sender.py:send():379] send: history +2024-12-11 11:38:47,751 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:38:47,759 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:38:47,939 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:47,939 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json +2024-12-11 11:38:47,940 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/1_eval_1_925210ff3b36d88f3887.mp4 +2024-12-11 11:38:47,940 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval +2024-12-11 11:38:47,988 INFO wandb-upload_0:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/algrlwjl-media/videos/eval/1_eval_1_925210ff3b36d88f3887.mp4 +2024-12-11 11:38:48,259 INFO wandb-upload_2:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/eex9yxjq-media/videos/eval/1_eval_1_925210ff3b36d88f3887.mp4 +2024-12-11 11:38:48,365 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:49,358 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:49,940 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:50,357 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:50,450 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:38:51,358 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:51,941 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:52,363 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:53,358 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:53,941 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:38:54,361 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:55,357 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:56,336 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:38:56,357 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:57,358 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:58,358 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:38:59,364 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:00,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:39:00,331 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:39:00,377 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:01,364 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:01,423 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:02,364 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:03,367 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:04,366 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:05,375 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:06,369 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:07,338 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:07,370 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:08,367 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:09,369 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:10,371 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:11,376 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:12,343 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:12,373 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:13,372 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:14,371 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:15,005 DEBUG SenderThread:743447 [sender.py:send():379] send: stats +2024-12-11 11:39:15,335 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:39:15,335 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:39:15,377 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:16,373 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:17,372 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:17,430 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:18,373 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:19,374 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:20,373 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:21,373 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:21,758 DEBUG SenderThread:743447 [sender.py:send():379] send: history +2024-12-11 11:39:21,759 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:39:21,759 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:39:21,950 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:39:21,950 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json +2024-12-11 11:39:21,951 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:22,200 INFO wandb-upload_0:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/wnjvdsbf-events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:39:22,374 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:23,170 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:23,373 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:23,951 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:24,374 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:25,374 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:25,951 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:26,373 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:27,373 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:27,952 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:28,298 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:28,375 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:29,375 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:29,953 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:30,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:39:30,332 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:39:30,374 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:31,374 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:31,954 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:32,375 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:33,322 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:33,375 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:33,954 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:34,376 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:35,375 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:35,955 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:36,375 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:37,376 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:37,956 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:38,344 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:38,376 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:39,377 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:39,957 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:40,376 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:41,376 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:41,958 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:42,376 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:43,376 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:43,454 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:43,958 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:44,376 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:45,005 DEBUG SenderThread:743447 [sender.py:send():379] send: stats +2024-12-11 11:39:45,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:39:45,332 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:39:45,377 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:45,959 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:46,377 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:47,377 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:47,960 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:48,377 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:48,576 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:49,377 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:49,961 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:50,377 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:51,378 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:51,961 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:52,378 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:53,378 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:53,803 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:53,962 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:54,378 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:55,378 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:55,963 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:56,378 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:57,378 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:57,964 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:39:58,378 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:58,900 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:39:59,378 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:39:59,964 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:00,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:40:00,332 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:40:00,378 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:01,379 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:01,965 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:02,384 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:02,517 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:40:02,518 DEBUG SenderThread:743447 [sender.py:send():379] send: files +2024-12-11 11:40:02,518 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_ddba0e3583786ac1097b.mp4 with policy now +2024-12-11 11:40:02,519 DEBUG SenderThread:743447 [sender.py:send():379] send: history +2024-12-11 11:40:02,519 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:40:02,519 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:40:02,735 INFO wandb-upload_2:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/2qt74j26-media/videos/eval/2_eval_2_ddba0e3583786ac1097b.mp4 +2024-12-11 11:40:02,965 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json +2024-12-11 11:40:02,966 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/2_eval_2_ddba0e3583786ac1097b.mp4 +2024-12-11 11:40:02,966 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval +2024-12-11 11:40:03,258 INFO wandb-upload_0:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/nmnl78ir-media/videos/eval/2_eval_2_ddba0e3583786ac1097b.mp4 +2024-12-11 11:40:03,390 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:03,966 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:04,387 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:04,529 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:40:05,387 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:05,966 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:06,386 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:07,389 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:08,386 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:09,394 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:09,533 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:40:09,621 DEBUG SenderThread:743447 [sender.py:send():379] send: history +2024-12-11 11:40:09,622 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:40:09,622 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:40:09,968 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:40:09,968 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json +2024-12-11 11:40:09,968 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:10,392 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:11,387 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:11,968 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:12,398 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:13,391 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:13,969 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:14,389 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:14,562 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:40:15,006 DEBUG SenderThread:743447 [sender.py:send():379] send: stats +2024-12-11 11:40:15,332 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:40:15,332 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:40:15,389 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:15,970 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:16,389 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:17,389 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:17,971 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:18,389 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:19,389 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:19,788 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:40:19,972 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:20,389 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:21,389 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:21,973 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:22,389 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:23,390 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:23,974 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:24,390 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:24,792 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:40:25,390 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:25,974 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:26,390 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:27,390 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:27,975 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:28,390 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:29,391 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:29,860 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:40:29,976 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:30,332 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:40:30,332 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:40:30,391 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:31,391 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:31,977 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:32,391 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:33,391 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:33,977 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:34,392 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:35,029 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:40:35,392 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:35,978 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:36,392 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:37,392 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:37,979 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:38,392 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:39,392 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:39,979 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:40,086 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:40:40,392 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:41,393 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:41,980 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:42,393 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:43,393 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:43,981 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:44,393 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:45,014 DEBUG SenderThread:743447 [sender.py:send():379] send: stats +2024-12-11 11:40:45,130 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:40:45,333 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:40:45,333 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:40:45,393 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:45,982 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:46,393 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:47,393 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:47,983 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:48,393 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:49,396 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:49,983 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:50,091 DEBUG SenderThread:743447 [sender.py:send():379] send: files +2024-12-11 11:40:50,091 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_b687720b2d32aeeb9a91.mp4 with policy now +2024-12-11 11:40:50,094 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:40:50,094 DEBUG SenderThread:743447 [sender.py:send():379] send: history +2024-12-11 11:40:50,095 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:40:50,105 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:40:50,133 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:40:50,329 INFO wandb-upload_2:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/6p2eraxb-media/videos/eval/3_eval_3_b687720b2d32aeeb9a91.mp4 +2024-12-11 11:40:50,394 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:50,987 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json +2024-12-11 11:40:50,987 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/3_eval_3_b687720b2d32aeeb9a91.mp4 +2024-12-11 11:40:50,987 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval +2024-12-11 11:40:51,327 INFO wandb-upload_0:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/aiy14acg-media/videos/eval/3_eval_3_b687720b2d32aeeb9a91.mp4 +2024-12-11 11:40:51,406 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:51,987 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:52,399 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:53,404 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:53,988 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:54,403 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:55,358 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:40:55,409 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:56,411 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:57,413 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:57,989 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:40:58,410 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:40:59,410 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:00,335 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:41:00,336 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:41:00,411 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:00,432 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:00,683 DEBUG SenderThread:743447 [sender.py:send():379] send: history +2024-12-11 11:41:00,683 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:41:00,683 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:41:00,991 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:41:00,991 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json +2024-12-11 11:41:01,267 INFO wandb-upload_2:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/8cztmbl3-events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:41:01,410 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:01,991 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:02,412 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:03,414 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:04,409 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:05,446 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:05,446 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:05,992 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:06,421 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:07,421 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:07,993 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:08,422 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:09,422 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:09,994 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:10,422 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:10,633 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:11,422 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:11,994 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:12,422 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:13,422 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:13,995 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:14,422 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:15,007 DEBUG SenderThread:743447 [sender.py:send():379] send: stats +2024-12-11 11:41:15,333 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:41:15,334 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:41:15,423 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:15,812 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:15,996 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:16,423 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:17,424 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:17,997 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:18,425 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:19,426 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:19,998 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:20,425 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:20,844 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:21,425 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:21,998 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:22,425 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:23,425 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:23,999 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:24,425 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:25,425 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:25,874 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:26,000 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:26,425 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:27,426 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:28,001 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:28,431 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:29,430 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:30,001 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:30,333 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:41:30,334 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:41:30,430 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:31,070 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:31,430 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:32,002 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:32,430 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:33,430 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:34,003 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:34,431 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:35,431 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:36,004 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:36,123 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:36,431 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:37,431 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:38,005 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:38,431 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:39,431 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:40,005 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:40,433 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:40,811 DEBUG SenderThread:743447 [sender.py:send():379] send: files +2024-12-11 11:41:40,811 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_10441113b4d3ea44e8c6.mp4 with policy now +2024-12-11 11:41:40,812 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:41:40,812 DEBUG SenderThread:743447 [sender.py:send():379] send: history +2024-12-11 11:41:40,812 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:41:40,820 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:41:41,006 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json +2024-12-11 11:41:41,006 INFO Thread-12 :743447 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/4_eval_4_10441113b4d3ea44e8c6.mp4 +2024-12-11 11:41:41,006 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval +2024-12-11 11:41:41,043 INFO wandb-upload_0:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/0somhq86-media/videos/eval/4_eval_4_10441113b4d3ea44e8c6.mp4 +2024-12-11 11:41:41,317 INFO wandb-upload_2:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/56mpvpsh-media/videos/eval/4_eval_4_10441113b4d3ea44e8c6.mp4 +2024-12-11 11:41:41,437 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:41,867 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:42,006 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:42,441 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:43,435 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:44,007 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:44,439 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:45,007 DEBUG SenderThread:743447 [sender.py:send():379] send: stats +2024-12-11 11:41:45,345 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:41:45,345 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:41:45,442 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:46,441 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:47,446 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:47,542 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:48,009 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:48,446 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:49,450 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:50,455 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:41:51,330 DEBUG SenderThread:743447 [sender.py:send():379] send: exit +2024-12-11 11:41:51,330 INFO SenderThread:743447 [sender.py:send_exit():586] handling exit code: 0 +2024-12-11 11:41:51,330 INFO SenderThread:743447 [sender.py:send_exit():588] handling runtime: 306 +2024-12-11 11:41:51,330 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:41:51,330 INFO SenderThread:743447 [sender.py:send_exit():594] send defer +2024-12-11 11:41:51,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:41:51,331 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-11 11:41:51,331 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:41:51,331 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-11 11:41:51,331 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 1 +2024-12-11 11:41:51,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:41:51,331 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-11 11:41:51,331 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:41:51,331 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-11 11:41:51,331 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 2 +2024-12-11 11:41:51,332 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:41:51,332 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-11 11:41:51,332 INFO HandlerThread:743447 [system_monitor.py:finish():203] Stopping system monitor +2024-12-11 11:41:51,332 DEBUG SystemMonitor:743447 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-11 11:41:51,332 INFO HandlerThread:743447 [interfaces.py:finish():200] Joined cpu monitor +2024-12-11 11:41:51,332 DEBUG SystemMonitor:743447 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-11 11:41:51,332 INFO HandlerThread:743447 [interfaces.py:finish():200] Joined disk monitor +2024-12-11 11:41:51,380 INFO HandlerThread:743447 [interfaces.py:finish():200] Joined gpu monitor +2024-12-11 11:41:51,381 INFO HandlerThread:743447 [interfaces.py:finish():200] Joined memory monitor +2024-12-11 11:41:51,381 INFO HandlerThread:743447 [interfaces.py:finish():200] Joined network monitor +2024-12-11 11:41:51,381 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:41:51,381 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-11 11:41:51,381 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 3 +2024-12-11 11:41:51,381 DEBUG SenderThread:743447 [sender.py:send():379] send: stats +2024-12-11 11:41:51,381 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:41:51,382 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-11 11:41:51,382 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:41:51,382 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-11 11:41:51,382 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 4 +2024-12-11 11:41:51,382 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:41:51,382 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-11 11:41:52,010 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:41:52,011 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json +2024-12-11 11:41:53,011 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:57,384 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:41:57,384 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:57,384 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:41:57,388 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: keepalive +2024-12-11 11:41:57,388 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-11 11:41:57,388 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 5 +2024-12-11 11:41:57,389 DEBUG SenderThread:743447 [sender.py:send():379] send: history +2024-12-11 11:41:57,389 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:41:57,389 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:41:57,389 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-11 11:41:57,389 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:41:57,389 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:41:57,390 DEBUG SenderThread:743447 [sender.py:send():379] send: history +2024-12-11 11:41:57,390 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:41:57,390 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:41:57,390 DEBUG SenderThread:743447 [sender.py:send():379] send: summary +2024-12-11 11:41:57,391 INFO SenderThread:743447 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:41:57,391 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:41:57,391 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-11 11:41:57,391 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 6 +2024-12-11 11:41:57,391 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:41:57,391 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-11 11:41:57,391 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:41:57,392 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-11 11:41:57,392 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 7 +2024-12-11 11:41:57,392 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:41:57,392 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:41:57,392 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-11 11:41:57,392 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:41:57,392 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-11 11:41:58,012 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json +2024-12-11 11:41:58,015 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 8 +2024-12-11 11:41:58,015 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:41:58,015 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-11 11:41:58,015 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:41:58,015 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-11 11:41:58,015 INFO SenderThread:743447 [job_builder.py:build():440] Attempting to build job artifact +2024-12-11 11:41:58,017 INFO SenderThread:743447 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-11 11:41:58,032 INFO SenderThread:743447 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-11 11:41:58,046 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 9 +2024-12-11 11:41:58,046 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:41:58,047 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-11 11:41:58,047 DEBUG SenderThread:743447 [sender.py:send():379] send: artifact +2024-12-11 11:41:58,331 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:41:58,562 INFO wandb-upload_1:743447 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpe19lto7x +2024-12-11 11:41:58,562 INFO wandb-upload_2:743447 [upload_job.py:push():85] Skipped uploading /tmp/tmpybw54_lu/wandb-job.json +2024-12-11 11:41:58,687 INFO wandb-upload_0:743447 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpugla6_vr +2024-12-11 11:41:59,013 INFO Thread-12 :743447 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:41:59,244 INFO SenderThread:743447 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM2NzYwNTg3Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM2NzYwMDc2Nw==', 'versionIndex': 364}}} +2024-12-11 11:41:59,245 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:41:59,245 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-11 11:41:59,245 INFO SenderThread:743447 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-11 11:42:00,013 INFO SenderThread:743447 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files +2024-12-11 11:42:00,013 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/config.yaml config.yaml +2024-12-11 11:42:00,014 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:42:00,015 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json wandb-summary.json +2024-12-11 11:42:00,015 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/requirements.txt requirements.txt +2024-12-11 11:42:00,016 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-metadata.json wandb-metadata.json +2024-12-11 11:42:00,016 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log output.log +2024-12-11 11:42:00,016 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/diff.patch diff.patch +2024-12-11 11:42:00,016 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-11 11:42:00,022 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/0_eval_0_dea3cb7ce16ad057bd5f.mp4 media/videos/eval/0_eval_0_dea3cb7ce16ad057bd5f.mp4 +2024-12-11 11:42:00,022 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/4_eval_4_10441113b4d3ea44e8c6.mp4 media/videos/eval/4_eval_4_10441113b4d3ea44e8c6.mp4 +2024-12-11 11:42:00,022 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/2_eval_2_ddba0e3583786ac1097b.mp4 media/videos/eval/2_eval_2_ddba0e3583786ac1097b.mp4 +2024-12-11 11:42:00,022 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/3_eval_3_b687720b2d32aeeb9a91.mp4 media/videos/eval/3_eval_3_b687720b2d32aeeb9a91.mp4 +2024-12-11 11:42:00,022 INFO SenderThread:743447 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/media/videos/eval/1_eval_1_925210ff3b36d88f3887.mp4 media/videos/eval/1_eval_1_925210ff3b36d88f3887.mp4 +2024-12-11 11:42:00,022 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 10 +2024-12-11 11:42:00,023 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:42:00,023 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:42:00,023 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-11 11:42:00,023 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:42:00,023 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-11 11:42:00,023 INFO SenderThread:743447 [file_pusher.py:finish():169] shutting down file pusher +2024-12-11 11:42:00,215 INFO wandb-upload_1:743447 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/config.yaml +2024-12-11 11:42:00,233 INFO wandb-upload_2:743447 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/wandb-summary.json +2024-12-11 11:42:00,247 INFO wandb-upload_3:743447 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/requirements.txt +2024-12-11 11:42:00,264 INFO wandb-upload_4:743447 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/files/output.log +2024-12-11 11:42:00,332 INFO wandb-upload_0:743447 [upload_job.py:push():130] Uploaded file /tmp/tmp0lcadrfewandb/o5djqyqe-events.out.tfevents.1733935005.leonmkim-ROG-Strix-G15CS-G15CS.743294.0 +2024-12-11 11:42:00,332 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:42:00,333 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:42:00,532 INFO Thread-11 (_thread_body):743447 [sender.py:transition_state():614] send defer: 11 +2024-12-11 11:42:00,533 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:42:00,533 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-11 11:42:00,533 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:42:00,533 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-11 11:42:00,533 INFO SenderThread:743447 [file_pusher.py:join():175] waiting for file pusher +2024-12-11 11:42:00,534 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 12 +2024-12-11 11:42:00,534 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:42:00,534 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-11 11:42:00,534 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:42:00,534 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-11 11:42:00,534 INFO SenderThread:743447 [file_stream.py:finish():601] file stream finish called +2024-12-11 11:42:00,730 INFO SenderThread:743447 [file_stream.py:finish():605] file stream finish is done +2024-12-11 11:42:00,731 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 13 +2024-12-11 11:42:00,731 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:42:00,731 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-11 11:42:00,731 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:42:00,731 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-11 11:42:00,731 INFO SenderThread:743447 [sender.py:transition_state():614] send defer: 14 +2024-12-11 11:42:00,731 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:42:00,731 DEBUG SenderThread:743447 [sender.py:send():379] send: final +2024-12-11 11:42:00,731 INFO HandlerThread:743447 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-11 11:42:00,731 DEBUG SenderThread:743447 [sender.py:send():379] send: footer +2024-12-11 11:42:00,731 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: defer +2024-12-11 11:42:00,731 INFO SenderThread:743447 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-11 11:42:00,732 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:42:00,732 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:42:00,732 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: server_info +2024-12-11 11:42:00,732 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: server_info +2024-12-11 11:42:00,734 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:42:00,734 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: get_summary +2024-12-11 11:42:00,734 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-11 11:42:00,735 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:42:00,795 DEBUG SenderThread:743447 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:42:00,795 INFO MainThread:743447 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-11 11:42:00,796 INFO MainThread:743447 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-11 11:42:00,796 INFO MainThread:743447 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-11 11:42:00,796 DEBUG HandlerThread:743447 [handler.py:handle_request():158] handle_request: shutdown +2024-12-11 11:42:00,796 INFO HandlerThread:743447 [handler.py:finish():882] shutting down handler +2024-12-11 11:42:01,734 INFO WriterThread:743447 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/run-ahosb0lo.wandb +2024-12-11 11:42:01,795 INFO SenderThread:743447 [sender.py:finish():1615] shutting down sender +2024-12-11 11:42:01,795 INFO SenderThread:743447 [file_pusher.py:finish():169] shutting down file pusher +2024-12-11 11:42:01,795 INFO SenderThread:743447 [file_pusher.py:join():175] waiting for file pusher diff --git a/113638/wandb/run-20241211_113644-ahosb0lo/logs/debug.log b/113638/wandb/run-20241211_113644-ahosb0lo/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d2fa4993882c1ade02b23b386433f0dd0e9e6a52 --- /dev/null +++ b/113638/wandb/run-20241211_113644-ahosb0lo/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-11 11:36:44,614 INFO MainThread:743294 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-11 11:36:44,614 INFO MainThread:743294 [wandb_setup.py:_flush():76] Configure stats pid to 743294 +2024-12-11 11:36:44,614 INFO MainThread:743294 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-11 11:36:44,614 INFO MainThread:743294 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/settings +2024-12-11 11:36:44,614 INFO MainThread:743294 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-11 11:36:44,614 INFO MainThread:743294 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-11 11:36:44,614 INFO MainThread:743294 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-11 11:36:44,614 INFO MainThread:743294 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-11 11:36:44,614 INFO MainThread:743294 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/logs/debug.log +2024-12-11 11:36:44,614 INFO MainThread:743294 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/wandb/run-20241211_113644-ahosb0lo/logs/debug-internal.log +2024-12-11 11:36:44,614 INFO MainThread:743294 [wandb_init.py:init():569] calling init triggers +2024-12-11 11:36:44,615 INFO MainThread:743294 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 2, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '265650', 'wandb_notes': '265650_req_265757restarted_2', 'eval': True, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim_left_to_right'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 16, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/265650', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'obs_shape': [14, 240, 320], 'action_shape': [36, 7], 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'true_action_history': False, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt'} +2024-12-11 11:36:44,615 INFO MainThread:743294 [wandb_init.py:init():619] starting backend +2024-12-11 11:36:44,615 INFO MainThread:743294 [wandb_init.py:init():623] setting up manager +2024-12-11 11:36:44,619 INFO MainThread:743294 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-11 11:36:44,620 INFO MainThread:743294 [wandb_init.py:init():631] backend started and connected +2024-12-11 11:36:44,632 INFO MainThread:743294 [wandb_init.py:init():720] updated telemetry +2024-12-11 11:36:44,639 INFO MainThread:743294 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-11 11:36:44,920 INFO MainThread:743294 [wandb_run.py:_on_init():2435] communicating current version +2024-12-11 11:36:44,991 INFO MainThread:743294 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.0 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-11 11:36:44,991 INFO MainThread:743294 [wandb_init.py:init():804] starting run threads in backend +2024-12-11 11:36:45,329 INFO MainThread:743294 [wandb_run.py:_console_start():2413] atexit reg +2024-12-11 11:36:45,329 INFO MainThread:743294 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-11 11:36:45,329 INFO MainThread:743294 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-11 11:36:45,329 INFO MainThread:743294 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-11 11:36:45,331 INFO MainThread:743294 [wandb_init.py:init():847] run started, returning control to user process +2024-12-11 11:36:45,331 INFO MainThread:743294 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265650/113638/tb, True +2024-12-11 11:36:49,564 INFO MainThread:743294 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-11 11:42:01,796 WARNING MsgRouterThr:743294 [router.py:message_loop():77] message_loop has been closed diff --git a/113638/wandb/run-20241211_113644-ahosb0lo/run-ahosb0lo.wandb b/113638/wandb/run-20241211_113644-ahosb0lo/run-ahosb0lo.wandb new file mode 100644 index 0000000000000000000000000000000000000000..d00935a48c723c27cac748929d115fccce0f3e1d --- /dev/null +++ b/113638/wandb/run-20241211_113644-ahosb0lo/run-ahosb0lo.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5a235a463c7c99af6fdcdad2bbd2c51fdc774401c7f98baa7f4c83783e6b5ec +size 294853 diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85f0762183c5c0c7a5e6fd848d5d1b1a3e998a51 --- /dev/null +++ b/config.yaml @@ -0,0 +1,453 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 0 +dataset_shuffle_seed: 2 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '265650' +wandb_notes: 265650_req_265757restarted_2 +eval: false +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: true +use_sdf_maps: true +use_normals_maps: true +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: +- hbm_twodim_left_to_right +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 16 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + obs_shape: + - 14 + - 240 + - 320 + action_shape: + - 36 + - 7 + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + policy_frequency: 5 + interpolation_frequency: 25 +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/snapshot_10500.pt b/snapshot_10500.pt new file mode 100644 index 0000000000000000000000000000000000000000..f21489566202e0f41c18ae0cb293bf439df7fc03 --- /dev/null +++ b/snapshot_10500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d240de137ad4b77d28e5670df4f74a7b7b6d8c14408e2a625ce14ced049ff3e8 +size 910445958 diff --git a/snapshot_10749.pt b/snapshot_10749.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad638e9f0cf2a4aedb195dfafe500fbb2e085c3d --- /dev/null +++ b/snapshot_10749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48677ce6dca1d2e08d5b6f808077e680050fad4ddf69814acc2224efe60c8ec4 +size 910445958 diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..a2274c899084938d0f0eaca72e7ef6980fee92b7 --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43310ae363123487086d4f0d99dcdb1c897916b9b111f7a5883c5f16bc55ec44 +size 910445958 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..c403c8a1ce182db4d63b6b212578d4e1919244e7 --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f355ea511588c624f1f12472f1c4f159ef0d5330732ad0eeeb11ce3dcda47e1 +size 910445958 diff --git a/snapshot_14453.pt b/snapshot_14453.pt new file mode 100644 index 0000000000000000000000000000000000000000..8bc1982c45eefbc6abb6d3b583f44eac03926103 --- /dev/null +++ b/snapshot_14453.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:246648ccf3ac9a0cc43cdf478d705e049c9d5b13df49f548b3e885ed1d078702 +size 910445958 diff --git a/snapshot_14999.pt b/snapshot_14999.pt new file mode 100644 index 0000000000000000000000000000000000000000..007e9e6e8ca1e77612e9f92e4f04fe8027f32ed2 --- /dev/null +++ b/snapshot_14999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98b56bd4a55998f86774e65d31ff27b1ee05ccce33d29fab015ccbba4c928ff5 +size 910445958 diff --git a/snapshot_1500.pt b/snapshot_1500.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a6fc500f1cc88c3453d6bf9cfb3f9eff2a7b587 --- /dev/null +++ b/snapshot_1500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb5da0e457c39a7fca91e2036bf75959fd2deb378f9a5bdfd8db9f650d6e2ee4 +size 910445448 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..e7d00bbbb30bd7ce72f0a90346362ad412cb4656 --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f06a576476429639f85664a8dac99b0ad1c299d48cf1688e0798331e10779556 +size 910445958 diff --git a/snapshot_3000.pt b/snapshot_3000.pt new file mode 100644 index 0000000000000000000000000000000000000000..f1ae36af63957bcdbf32b7bb2df08c2e1aea67e0 --- /dev/null +++ b/snapshot_3000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40adcddb32ad7565c27bbb7931b55d1bdb81a60711c2dc44147e3e6c14c1cf0d +size 910445448 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..4e6b1557486bf7905d9f44669ecee1e2131031e1 --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddd3dc4cb211efc6b103d6fa3fb8bd1e8d4476431808df4fefaace7407f10e39 +size 910445448 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ddd1f2af1b9c49dd5e6011ef1bbe659b67c45f6 --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a1815afe6e5c1a74b648c4a20d28bc82d2ce774a09d2f6e40add960ef7a5087 +size 910445958 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..cbbed51e21e1c6c2da06e28f539c057250b905ac --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a262872a96cb9a84a253b761777f3343c3be44ab6d407dd5c0f6618e787f92dd +size 910445958 diff --git a/snapshot_7749.pt b/snapshot_7749.pt new file mode 100644 index 0000000000000000000000000000000000000000..d8b9ee786ab28be1281e928d8139a6128008b28a --- /dev/null +++ b/snapshot_7749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d756018a427a3512d46fcac64ca6b9cc1cab0a7e72647fa23cc7d81d52904927 +size 910445958 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..01ee122f489cfc5808cdd4017d5c0a863f7fa8ab --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a3c348c4dc3a78f448920e855df075f024e73684fa52b7f09d95b36c6c62cd2 +size 910445958 diff --git a/snapshot_9749.pt b/snapshot_9749.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a9907c6c50e715fed9a7acba5a3dd934d581311 --- /dev/null +++ b/snapshot_9749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f923ae588a46d26977bf92ae2e7ebec41e48ec0d7c589fb6c73da94360c81915 +size 910445958 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..e7d00bbbb30bd7ce72f0a90346362ad412cb4656 --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f06a576476429639f85664a8dac99b0ad1c299d48cf1688e0798331e10779556 +size 910445958