diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..b1bfce6b5ac48aca8d063fd67729e34d3fd9837d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,19 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +110822/wandb/run-20241211_110828-7k8jf8l5/run-7k8jf8l5.wandb filter=lfs diff=lfs merge=lfs -text +110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/1_eval_1_dce37a039a0f3ae510e5.mp4 filter=lfs diff=lfs merge=lfs -text +110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/0_eval_0_a7ce057e6fe1a72a45a4.mp4 filter=lfs diff=lfs merge=lfs -text +110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/4_eval_4_b9f6613e7cd6916b7519.mp4 filter=lfs diff=lfs merge=lfs -text +110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/2_eval_2_c7cd658cf3e0e8f2dada.mp4 filter=lfs diff=lfs merge=lfs -text +110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/3_eval_3_ac712fc970ab6ee59b1c.mp4 filter=lfs diff=lfs merge=lfs -text +110822/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +110822/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +110822/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +110822/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +110822/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +110822/episode_rosbags/episode_2_2024-12-11-11-10-28.bag filter=lfs diff=lfs merge=lfs -text +110822/episode_rosbags/episode_1_2024-12-11-11-09-42.bag filter=lfs diff=lfs merge=lfs -text +110822/episode_rosbags/episode_4_2024-12-11-11-12-05.bag filter=lfs diff=lfs merge=lfs -text +110822/episode_rosbags/episode_0_2024-12-11-11-08-46.bag filter=lfs diff=lfs merge=lfs -text +110822/episode_rosbags/episode_3_2024-12-11-11-11-21.bag filter=lfs diff=lfs merge=lfs -text diff --git a/110822/.hydra/config.yaml b/110822/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7bfe2ee8524fb27ee4eee1e364e87232c71c260 --- /dev/null +++ b/110822/.hydra/config.yaml @@ -0,0 +1,349 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '265644' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/110822/.hydra/hydra.yaml b/110822/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f48d7a0fc517e7a1c31ba2ef98e22f20596c24c7 --- /dev/null +++ b/110822/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${experiment_dir}/${now:%H%M%S} + sweep: + dir: ${experiment_dir}/${now:%H%M%S} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${experiment_dir}/${now:%H%M%S}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/110822/.hydra/overrides.yaml b/110822/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/110822/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/110822/episode_rosbags/aligned_depth_to_color_K.npy b/110822/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/110822/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/110822/episode_rosbags/cam_tf_world.npy b/110822/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/110822/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/110822/episode_rosbags/color_K.npy b/110822/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/110822/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/110822/episode_rosbags/depth_K.npy b/110822/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/110822/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/110822/episode_rosbags/episode_0_2024-12-11-11-08-46.bag b/110822/episode_rosbags/episode_0_2024-12-11-11-08-46.bag new file mode 100644 index 0000000000000000000000000000000000000000..0de2f55c6c264c109913b2e9804d256064548bbb --- /dev/null +++ b/110822/episode_rosbags/episode_0_2024-12-11-11-08-46.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6e5129d33fd8b4d37ab3a890502708725cb34baf12a8c8b9717f238851cf052 +size 1441910689 diff --git a/110822/episode_rosbags/episode_1_2024-12-11-11-09-42.bag b/110822/episode_rosbags/episode_1_2024-12-11-11-09-42.bag new file mode 100644 index 0000000000000000000000000000000000000000..c0b334e80fb2b39695ac21324c3c2376b790c255 --- /dev/null +++ b/110822/episode_rosbags/episode_1_2024-12-11-11-09-42.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2433a40507fa44d741c7e88c1b40dd260cca8afb2191fb8e202902e350489acf +size 1440811825 diff --git a/110822/episode_rosbags/episode_2_2024-12-11-11-10-28.bag b/110822/episode_rosbags/episode_2_2024-12-11-11-10-28.bag new file mode 100644 index 0000000000000000000000000000000000000000..1e26d6435cae1a71739da9cc9473cd42ea38aa84 --- /dev/null +++ b/110822/episode_rosbags/episode_2_2024-12-11-11-10-28.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87a917ec6788ed58bb04002cb72301fca0d24fc04c5a105d17263dd2895f8b97 +size 1445451697 diff --git a/110822/episode_rosbags/episode_3_2024-12-11-11-11-21.bag b/110822/episode_rosbags/episode_3_2024-12-11-11-11-21.bag new file mode 100644 index 0000000000000000000000000000000000000000..e909976497dcf2fdac1bdac100f8316a1e54282e --- /dev/null +++ b/110822/episode_rosbags/episode_3_2024-12-11-11-11-21.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74c984c60f941131a0301aad7315d0a87d4a46d4825a03d896cd9c1badb863a3 +size 1461872966 diff --git a/110822/episode_rosbags/episode_4_2024-12-11-11-12-05.bag b/110822/episode_rosbags/episode_4_2024-12-11-11-12-05.bag new file mode 100644 index 0000000000000000000000000000000000000000..312e88b22230165e660e588ee44dc754a2bd3ae0 --- /dev/null +++ b/110822/episode_rosbags/episode_4_2024-12-11-11-12-05.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdf7a754dd107e6bc2d430943b45e69456368bb69b8103e8c5b234a6b121c254 +size 1438750631 diff --git a/110822/eval_policy.log b/110822/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..616bdd3957db2aa85d299b8d6c1337cb82d0a0b6 --- /dev/null +++ b/110822/eval_policy.log @@ -0,0 +1,15 @@ +[2024-12-11 11:08:22,480][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-11 11:08:22,489][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2024-12-11 11:08:24,122][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/110822/eval_robot.log b/110822/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..a3a00aef4903713265506c2c17068197f3fd80d0 --- /dev/null +++ b/110822/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-11 11:08:22,631][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-11 11:08:22,634][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/110822/eval_video/0_eval.mp4 b/110822/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8c3fa49623844076f5e07b5b66b2d228e30bda00 --- /dev/null +++ b/110822/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7ce057e6fe1a72a45a4317170a65df82df6555562cdad1d7cd7243949470e1a +size 1148214 diff --git a/110822/eval_video/1_eval.mp4 b/110822/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df434b685b801cad39682ce3553022a610ec95f1 --- /dev/null +++ b/110822/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dce37a039a0f3ae510e5dcc44fee8b8ef5b47c9b606de8657c821538ad6b0900 +size 1152733 diff --git a/110822/eval_video/2_eval.mp4 b/110822/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ac3e90e1bbd9c99da70cf946e433a873ac4a54d1 --- /dev/null +++ b/110822/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7cd658cf3e0e8f2dada2bbb69769df6b96cb1af52105e6b852a5aa6e7dce071 +size 1195653 diff --git a/110822/eval_video/3_eval.mp4 b/110822/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3aa5b9dc182b9d9b39e55098744684408ca0c7a8 --- /dev/null +++ b/110822/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac712fc970ab6ee59b1c4c0c66fe4bb4b0ecb93b156e194c41d4bf46045d2870 +size 1242757 diff --git a/110822/eval_video/4_eval.mp4 b/110822/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d17c4d7e83ed371aa916136ad4a7e5b1a3e6ba4b --- /dev/null +++ b/110822/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9f6613e7cd6916b75191e5752ebdf1ff8273ec0404a1b2cc3197bf4e1dbcc81 +size 1261098 diff --git a/110822/tb/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 b/110822/tb/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 new file mode 100644 index 0000000000000000000000000000000000000000..d9b511fb0ed60d8879ce8358b9719fac25289bf4 --- /dev/null +++ b/110822/tb/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c5470fc2eebf745b319529f5660714b91d49e811b7635e9dd9d2e3fb25b23a8 +size 1103 diff --git a/110822/wandb/debug-internal.log b/110822/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..42926a5950f23efcaa809f2543c32bb089dfb298 --- /dev/null +++ b/110822/wandb/debug-internal.log @@ -0,0 +1,758 @@ +2024-12-11 11:08:28,505 INFO StreamThr :736906 [internal.py:wandb_internal():85] W&B internal server running at pid: 736906, started at: 2024-12-11 11:08:28.505002 +2024-12-11 11:08:28,506 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status +2024-12-11 11:08:28,525 INFO WriterThread:736906 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/run-7k8jf8l5.wandb +2024-12-11 11:08:28,525 DEBUG SenderThread:736906 [sender.py:send():379] send: header +2024-12-11 11:08:28,533 DEBUG SenderThread:736906 [sender.py:send():379] send: run +2024-12-11 11:08:28,813 INFO SenderThread:736906 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files +2024-12-11 11:08:28,813 INFO SenderThread:736906 [sender.py:_start_run_threads():1188] run started: 7k8jf8l5 with start time 1733933308.5053 +2024-12-11 11:08:28,832 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: check_version +2024-12-11 11:08:28,833 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: check_version +2024-12-11 11:08:28,887 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: run_start +2024-12-11 11:08:28,891 DEBUG HandlerThread:736906 [system_info.py:__init__():26] System info init +2024-12-11 11:08:28,892 DEBUG HandlerThread:736906 [system_info.py:__init__():41] System info init done +2024-12-11 11:08:28,892 INFO HandlerThread:736906 [system_monitor.py:start():194] Starting system monitor +2024-12-11 11:08:28,892 INFO SystemMonitor:736906 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-11 11:08:28,892 INFO HandlerThread:736906 [system_monitor.py:probe():214] Collecting system info +2024-12-11 11:08:28,893 INFO SystemMonitor:736906 [interfaces.py:start():188] Started cpu monitoring +2024-12-11 11:08:28,893 INFO SystemMonitor:736906 [interfaces.py:start():188] Started disk monitoring +2024-12-11 11:08:28,894 INFO SystemMonitor:736906 [interfaces.py:start():188] Started gpu monitoring +2024-12-11 11:08:28,894 INFO SystemMonitor:736906 [interfaces.py:start():188] Started memory monitoring +2024-12-11 11:08:28,894 INFO SystemMonitor:736906 [interfaces.py:start():188] Started network monitoring +2024-12-11 11:08:28,905 DEBUG HandlerThread:736906 [system_info.py:probe():152] Probing system +2024-12-11 11:08:28,907 DEBUG HandlerThread:736906 [system_info.py:_probe_git():137] Probing git +2024-12-11 11:08:28,913 DEBUG HandlerThread:736906 [system_info.py:_probe_git():145] Probing git done +2024-12-11 11:08:28,913 DEBUG HandlerThread:736906 [system_info.py:probe():200] Probing system done +2024-12-11 11:08:28,913 DEBUG HandlerThread:736906 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-11T16:08:28.905581', 'startedAt': '2024-12-11T16:08:28.497858', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '1b557f164c7f62a8d1545c761b13ede1677c3a83'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.98675, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.916, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.021, 'min': 800.0, 'max': 4700.0}, {'current': 4500.003, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 623.7553443908691}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71596908569336}} +2024-12-11 11:08:28,913 INFO HandlerThread:736906 [system_monitor.py:probe():224] Finished collecting system info +2024-12-11 11:08:28,913 INFO HandlerThread:736906 [system_monitor.py:probe():227] Publishing system info +2024-12-11 11:08:28,913 DEBUG HandlerThread:736906 [system_info.py:_save_code():44] Saving code +2024-12-11 11:08:28,915 DEBUG HandlerThread:736906 [system_info.py:_save_code():67] Saving code done +2024-12-11 11:08:28,915 DEBUG HandlerThread:736906 [system_info.py:_save_patches():84] Saving git patches +2024-12-11 11:08:28,980 DEBUG HandlerThread:736906 [system_info.py:_save_patches():126] Saving git patches done +2024-12-11 11:08:28,981 INFO HandlerThread:736906 [system_monitor.py:probe():229] Finished publishing system info +2024-12-11 11:08:28,984 DEBUG SenderThread:736906 [sender.py:send():379] send: files +2024-12-11 11:08:28,984 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-11 11:08:28,984 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-11 11:08:28,984 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-11 11:08:29,219 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: python_packages +2024-12-11 11:08:29,220 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: python_packages +2024-12-11 11:08:29,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:08:29,224 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:29,225 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:08:29,227 INFO HandlerThread:736906 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/tb" + save: true +} +control { + relay_id: "4910867fa5cc4067977e0abd74cdd7a1" +} +_info { + stream_id: "7k8jf8l5" +} + +2024-12-11 11:08:29,227 DEBUG HandlerThread:736906 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-11 11:08:29,233 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/x1l1f2rp-wandb-metadata.json +2024-12-11 11:08:29,271 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/i9e9567q-diff.patch +2024-12-11 11:08:29,293 DEBUG SenderThread:736906 [sender.py:send():379] send: telemetry +2024-12-11 11:08:29,293 DEBUG SenderThread:736906 [sender.py:send():379] send: tbrecord +2024-12-11 11:08:29,294 DEBUG SenderThread:736906 [sender.py:send():379] send: files +2024-12-11 11:08:29,294 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 with policy live +2024-12-11 11:08:29,297 INFO wandb-upload_1:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/rdhewrwt-code/FISH/eval_robot.py +2024-12-11 11:08:29,490 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/jpf62kj4-events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:08:29,814 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-metadata.json +2024-12-11 11:08:29,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/requirements.txt +2024-12-11 11:08:29,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/diff.patch +2024-12-11 11:08:29,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:29,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:08:29,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/code/FISH/eval_robot.py +2024-12-11 11:08:29,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/code +2024-12-11 11:08:29,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/code/FISH +2024-12-11 11:08:30,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:31,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:31,814 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:31,970 DEBUG SenderThread:736906 [sender.py:send():379] send: config +2024-12-11 11:08:32,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:33,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:33,814 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:34,030 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:08:34,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:35,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:35,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:36,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:37,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:37,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:38,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:39,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:08:39,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:40,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:41,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:41,817 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:42,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:43,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:44,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:08:44,220 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:08:44,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:44,316 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:08:45,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:45,818 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:46,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:47,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:48,223 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:49,223 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:49,344 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:08:49,819 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:50,223 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:51,223 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:51,819 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:52,223 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:53,224 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:53,820 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:54,224 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:54,424 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:08:55,224 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:55,820 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:56,224 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:57,224 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:57,821 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:58,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:59,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:08:59,220 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:08:59,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:59,437 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:08:59,822 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:59,822 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/config.yaml +2024-12-11 11:09:00,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:01,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:01,822 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:02,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:03,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:03,823 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:04,265 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:04,625 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:05,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:05,823 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:06,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:07,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:07,824 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:08,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:09,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:09,626 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:09,825 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:10,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:11,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:11,825 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:12,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:13,234 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:13,826 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:14,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:09:14,220 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:09:14,235 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:15,202 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:15,235 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:15,461 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:09:15,462 DEBUG SenderThread:736906 [sender.py:send():379] send: files +2024-12-11 11:09:15,463 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_a7ce057e6fe1a72a45a4.mp4 with policy now +2024-12-11 11:09:15,463 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:09:15,463 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:09:15,463 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:09:15,737 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/qmodnrad-media/videos/eval/0_eval_0_a7ce057e6fe1a72a45a4.mp4 +2024-12-11 11:09:15,826 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:15,827 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:09:15,827 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/0_eval_0_a7ce057e6fe1a72a45a4.mp4 +2024-12-11 11:09:15,827 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media +2024-12-11 11:09:15,827 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos +2024-12-11 11:09:15,827 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval +2024-12-11 11:09:16,102 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/9jxf4xwe-media/videos/eval/0_eval_0_a7ce057e6fe1a72a45a4.mp4 +2024-12-11 11:09:16,235 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:17,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:17,827 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:18,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:19,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:19,828 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:20,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:20,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:21,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:22,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:23,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:23,829 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:24,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:25,237 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:26,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:26,237 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:27,237 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:28,237 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:28,894 DEBUG SystemMonitor:736906 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-11 11:09:28,895 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:09:29,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:09:29,220 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:09:29,261 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:30,237 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:31,237 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:31,332 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:32,237 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:33,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:34,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:35,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:36,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:37,223 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:37,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:38,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:38,834 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:09:39,088 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/xszj87dx-events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:09:39,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:39,835 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:40,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:41,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:41,835 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:42,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:42,452 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:43,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:43,836 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:44,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:09:44,221 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:09:44,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:45,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:45,837 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:46,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:47,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:47,554 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:47,838 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:48,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:49,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:49,838 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:50,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:51,240 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:51,839 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:52,240 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:52,750 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:53,240 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:53,840 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:54,240 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:55,240 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:55,840 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:56,241 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:57,243 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:57,841 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:57,956 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:58,244 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:58,895 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:09:59,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:09:59,221 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:09:59,243 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:59,842 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:00,243 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:01,243 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:01,843 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:02,243 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:03,148 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:03,243 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:03,843 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:04,244 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:05,244 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:05,844 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:06,244 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:07,244 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:07,845 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:08,244 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:08,347 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:09,245 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:09,845 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:10,245 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:11,251 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:11,425 DEBUG SenderThread:736906 [sender.py:send():379] send: files +2024-12-11 11:10:11,425 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_dce37a039a0f3ae510e5.mp4 with policy now +2024-12-11 11:10:11,426 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:10:11,426 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:10:11,427 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:10:11,429 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:10:11,681 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/vbziq3qb-media/videos/eval/1_eval_1_dce37a039a0f3ae510e5.mp4 +2024-12-11 11:10:11,846 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:10:11,846 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:11,846 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/1_eval_1_dce37a039a0f3ae510e5.mp4 +2024-12-11 11:10:11,847 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval +2024-12-11 11:10:12,117 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/te2j2jvb-media/videos/eval/1_eval_1_dce37a039a0f3ae510e5.mp4 +2024-12-11 11:10:12,253 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:13,255 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:13,444 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:13,847 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:14,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:10:14,227 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:10:14,269 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:15,256 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:15,847 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:16,258 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:17,253 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:17,848 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:18,256 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:19,074 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:19,254 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:20,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:21,258 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:22,256 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:23,257 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:24,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:24,257 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:25,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:25,624 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:10:25,624 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:10:25,625 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:10:25,851 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:10:25,851 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:10:26,262 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:27,262 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:27,851 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:28,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:28,896 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:10:29,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:10:29,222 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:10:29,259 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:29,343 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:29,852 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:30,259 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:31,259 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:31,852 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:32,259 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:33,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:33,853 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:34,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:34,484 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:35,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:35,854 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:36,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:37,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:37,855 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:38,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:39,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:39,684 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:39,855 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:40,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:41,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:41,856 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:42,261 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:43,261 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:43,857 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:44,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:10:44,222 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:10:44,261 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:44,710 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:45,261 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:45,858 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:46,261 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:47,261 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:47,858 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:48,265 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:49,262 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:49,859 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:49,896 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:50,262 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:51,262 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:51,860 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:52,263 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:53,263 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:53,861 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:54,263 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:55,085 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:55,264 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:55,861 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:56,264 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:57,264 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:57,862 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:58,266 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:58,595 DEBUG SenderThread:736906 [sender.py:send():379] send: files +2024-12-11 11:10:58,595 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_c7cd658cf3e0e8f2dada.mp4 with policy now +2024-12-11 11:10:58,597 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:10:58,598 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:10:58,598 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:10:58,598 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:10:58,862 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:10:58,863 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/2_eval_2_c7cd658cf3e0e8f2dada.mp4 +2024-12-11 11:10:58,863 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval +2024-12-11 11:10:58,869 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/u7agmibs-media/videos/eval/2_eval_2_c7cd658cf3e0e8f2dada.mp4 +2024-12-11 11:10:58,896 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:10:59,153 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/veshwbiw-media/videos/eval/2_eval_2_c7cd658cf3e0e8f2dada.mp4 +2024-12-11 11:10:59,224 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:10:59,225 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:10:59,270 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:59,863 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:00,269 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:00,392 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:01,267 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:02,269 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:03,269 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:03,864 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:04,305 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:05,278 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:05,865 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:06,273 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:06,306 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:07,276 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:08,277 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:09,275 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:10,274 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:11,277 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:12,232 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:12,278 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:13,274 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:14,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:11:14,227 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:11:14,277 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:15,281 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:16,280 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:17,282 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:17,311 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:17,475 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:11:17,475 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:11:17,475 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:11:17,868 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:11:17,869 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:11:18,141 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/wagm2b1h-events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:11:18,283 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:19,288 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:19,869 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:20,287 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:21,285 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:21,870 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:22,285 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:22,473 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:23,282 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:23,871 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:24,301 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:25,284 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:25,871 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:26,282 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:27,292 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:27,477 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:27,872 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:28,283 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:28,897 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:11:29,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:11:29,227 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:11:29,283 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:29,873 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:30,285 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:31,284 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:31,873 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:32,284 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:32,506 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:33,286 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:33,874 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:34,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:35,288 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:35,875 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:36,291 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:37,291 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:37,511 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:37,875 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:38,289 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:39,288 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:39,876 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:40,290 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:41,290 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:41,877 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:42,291 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:42,593 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:43,292 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:43,878 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:44,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:11:44,226 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:11:44,293 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:45,292 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:45,880 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:46,292 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:47,294 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:47,631 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:47,880 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:48,293 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:49,329 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:49,881 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:50,298 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:51,303 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:51,882 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:52,314 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:52,331 DEBUG SenderThread:736906 [sender.py:send():379] send: files +2024-12-11 11:11:52,331 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_ac712fc970ab6ee59b1c.mp4 with policy now +2024-12-11 11:11:52,377 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:11:52,379 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:11:52,379 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:11:52,380 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:11:52,610 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/yfw8lnde-media/videos/eval/3_eval_3_ac712fc970ab6ee59b1c.mp4 +2024-12-11 11:11:52,883 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:11:52,883 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/3_eval_3_ac712fc970ab6ee59b1c.mp4 +2024-12-11 11:11:52,883 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval +2024-12-11 11:11:53,178 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/j207a1ci-media/videos/eval/3_eval_3_ac712fc970ab6ee59b1c.mp4 +2024-12-11 11:11:53,311 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:53,382 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:53,883 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:54,306 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:55,311 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:55,884 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:56,314 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:57,311 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:58,309 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:58,897 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:11:58,898 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:59,229 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:11:59,230 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:11:59,316 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:59,885 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:00,317 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:01,318 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:01,886 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:12:02,316 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:02,525 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:12:02,526 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:12:02,526 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:12:02,886 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:12:03,325 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:03,886 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:04,243 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:04,323 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:05,322 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:06,324 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:07,322 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:07,888 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:08,320 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:09,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:09,355 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:09,888 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:10,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:11,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:11,889 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:12,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:13,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:13,890 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:14,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:12:14,227 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:12:14,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:14,382 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:15,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:15,891 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:16,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:17,322 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:17,892 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:18,323 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:19,322 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:19,567 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:19,892 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:20,326 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:21,326 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:21,893 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:22,323 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:23,323 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:23,894 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:24,323 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:24,575 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:25,324 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:25,894 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:26,324 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:27,325 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:27,895 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:28,325 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:28,898 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:12:29,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:12:29,227 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:12:29,325 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:29,783 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:29,896 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:30,325 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:31,326 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:31,896 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:32,326 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:33,326 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:33,897 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:34,326 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:34,978 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:35,326 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:35,898 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:36,334 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:36,910 DEBUG SenderThread:736906 [sender.py:send():379] send: files +2024-12-11 11:12:36,910 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_b9f6613e7cd6916b7519.mp4 with policy now +2024-12-11 11:12:36,913 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:12:36,913 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:12:36,913 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:12:36,914 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:12:37,174 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/5dcy7vyd-media/videos/eval/4_eval_4_b9f6613e7cd6916b7519.mp4 +2024-12-11 11:12:37,337 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:37,899 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:12:37,899 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:37,899 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/4_eval_4_b9f6613e7cd6916b7519.mp4 +2024-12-11 11:12:37,899 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval +2024-12-11 11:12:38,192 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/663gsrmo-media/videos/eval/4_eval_4_b9f6613e7cd6916b7519.mp4 +2024-12-11 11:12:38,346 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:39,333 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:40,341 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:40,890 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:41,334 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:41,900 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:42,334 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:43,344 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:43,901 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:44,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:12:44,239 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:12:44,344 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:45,348 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:45,454 DEBUG SenderThread:736906 [sender.py:send():379] send: exit +2024-12-11 11:12:45,454 INFO SenderThread:736906 [sender.py:send_exit():586] handling exit code: 0 +2024-12-11 11:12:45,454 INFO SenderThread:736906 [sender.py:send_exit():588] handling runtime: 256 +2024-12-11 11:12:45,488 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:12:45,489 INFO SenderThread:736906 [sender.py:send_exit():594] send defer +2024-12-11 11:12:45,489 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:45,489 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-11 11:12:45,489 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:45,489 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-11 11:12:45,489 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 1 +2024-12-11 11:12:45,489 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:45,489 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-11 11:12:45,489 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:45,489 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-11 11:12:45,489 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 2 +2024-12-11 11:12:45,489 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:45,489 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-11 11:12:45,489 INFO HandlerThread:736906 [system_monitor.py:finish():203] Stopping system monitor +2024-12-11 11:12:45,490 DEBUG SystemMonitor:736906 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-11 11:12:45,490 DEBUG SystemMonitor:736906 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-11 11:12:45,490 INFO HandlerThread:736906 [interfaces.py:finish():200] Joined cpu monitor +2024-12-11 11:12:45,490 INFO HandlerThread:736906 [interfaces.py:finish():200] Joined disk monitor +2024-12-11 11:12:45,539 INFO HandlerThread:736906 [interfaces.py:finish():200] Joined gpu monitor +2024-12-11 11:12:45,540 INFO HandlerThread:736906 [interfaces.py:finish():200] Joined memory monitor +2024-12-11 11:12:45,540 INFO HandlerThread:736906 [interfaces.py:finish():200] Joined network monitor +2024-12-11 11:12:45,540 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:45,540 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-11 11:12:45,540 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 3 +2024-12-11 11:12:45,540 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:12:45,540 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:45,541 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-11 11:12:45,541 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:45,541 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-11 11:12:45,541 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 4 +2024-12-11 11:12:45,541 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:45,541 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-11 11:12:45,901 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:12:45,902 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:45,902 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:12:46,159 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/f6t033zr-events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:12:51,543 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:12:51,543 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:51,543 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:51,543 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: keepalive +2024-12-11 11:12:51,543 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-11 11:12:51,543 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:51,543 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 5 +2024-12-11 11:12:51,544 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:12:51,544 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:51,544 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:12:51,545 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-11 11:12:51,545 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:12:51,546 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:12:51,546 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:12:51,546 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:12:51,549 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:12:51,550 DEBUG SenderThread:736906 [sender.py:send():379] send: summary +2024-12-11 11:12:51,550 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:12:51,550 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:51,550 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-11 11:12:51,550 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 6 +2024-12-11 11:12:51,550 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:51,550 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-11 11:12:51,550 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:51,550 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-11 11:12:51,550 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 7 +2024-12-11 11:12:51,551 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:51,551 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:51,551 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-11 11:12:51,551 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:51,551 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-11 11:12:51,903 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:12:52,455 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:12:53,512 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 8 +2024-12-11 11:12:53,512 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:12:53,512 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:53,512 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-11 11:12:53,512 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:53,512 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-11 11:12:53,512 INFO SenderThread:736906 [job_builder.py:build():440] Attempting to build job artifact +2024-12-11 11:12:53,515 INFO SenderThread:736906 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-11 11:12:53,530 INFO SenderThread:736906 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-11 11:12:53,545 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 9 +2024-12-11 11:12:53,545 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:53,545 DEBUG SenderThread:736906 [sender.py:send():379] send: artifact +2024-12-11 11:12:53,545 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-11 11:12:53,904 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:54,052 INFO wandb-upload_1:736906 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpym6m9k3x +2024-12-11 11:12:54,052 INFO wandb-upload_2:736906 [upload_job.py:push():85] Skipped uploading /tmp/tmpq1auzwlr/wandb-job.json +2024-12-11 11:12:54,169 INFO wandb-upload_0:736906 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp9l1c58z3 +2024-12-11 11:12:54,456 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:12:54,800 INFO SenderThread:736906 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM2NzU4MjI2NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM2NzU3NzcyOA==', 'versionIndex': 360}}} +2024-12-11 11:12:54,800 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:54,800 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-11 11:12:54,800 INFO SenderThread:736906 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-11 11:12:54,905 INFO SenderThread:736906 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files +2024-12-11 11:12:54,905 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/config.yaml config.yaml +2024-12-11 11:12:54,905 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json wandb-summary.json +2024-12-11 11:12:54,905 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/requirements.txt requirements.txt +2024-12-11 11:12:54,906 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-metadata.json wandb-metadata.json +2024-12-11 11:12:54,908 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:12:54,909 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log output.log +2024-12-11 11:12:54,910 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/diff.patch diff.patch +2024-12-11 11:12:54,911 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-11 11:12:54,911 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/1_eval_1_dce37a039a0f3ae510e5.mp4 media/videos/eval/1_eval_1_dce37a039a0f3ae510e5.mp4 +2024-12-11 11:12:54,911 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/2_eval_2_c7cd658cf3e0e8f2dada.mp4 media/videos/eval/2_eval_2_c7cd658cf3e0e8f2dada.mp4 +2024-12-11 11:12:54,912 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/4_eval_4_b9f6613e7cd6916b7519.mp4 media/videos/eval/4_eval_4_b9f6613e7cd6916b7519.mp4 +2024-12-11 11:12:54,912 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/0_eval_0_a7ce057e6fe1a72a45a4.mp4 media/videos/eval/0_eval_0_a7ce057e6fe1a72a45a4.mp4 +2024-12-11 11:12:54,912 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/3_eval_3_ac712fc970ab6ee59b1c.mp4 media/videos/eval/3_eval_3_ac712fc970ab6ee59b1c.mp4 +2024-12-11 11:12:54,912 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 10 +2024-12-11 11:12:54,912 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:12:54,912 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:54,912 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-11 11:12:54,912 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:54,912 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-11 11:12:54,912 INFO SenderThread:736906 [file_pusher.py:finish():169] shutting down file pusher +2024-12-11 11:12:55,091 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/requirements.txt +2024-12-11 11:12:55,126 INFO wandb-upload_1:736906 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/config.yaml +2024-12-11 11:12:55,127 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:12:55,143 INFO wandb-upload_3:736906 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:55,344 INFO Thread-11 (_thread_body):736906 [sender.py:transition_state():614] send defer: 11 +2024-12-11 11:12:55,344 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:55,344 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-11 11:12:55,344 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:55,344 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-11 11:12:55,344 INFO SenderThread:736906 [file_pusher.py:join():175] waiting for file pusher +2024-12-11 11:12:55,345 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 12 +2024-12-11 11:12:55,345 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:55,345 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-11 11:12:55,345 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:55,345 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-11 11:12:55,346 INFO SenderThread:736906 [file_stream.py:finish():601] file stream finish called +2024-12-11 11:12:55,456 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:12:55,571 INFO SenderThread:736906 [file_stream.py:finish():605] file stream finish is done +2024-12-11 11:12:55,571 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 13 +2024-12-11 11:12:55,571 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:12:55,572 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:55,572 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-11 11:12:55,572 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:55,572 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-11 11:12:55,572 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 14 +2024-12-11 11:12:55,572 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:55,572 DEBUG SenderThread:736906 [sender.py:send():379] send: final +2024-12-11 11:12:55,572 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-11 11:12:55,572 DEBUG SenderThread:736906 [sender.py:send():379] send: footer +2024-12-11 11:12:55,572 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:55,573 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-11 11:12:55,573 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:12:55,573 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:12:55,573 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: server_info +2024-12-11 11:12:55,573 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: server_info +2024-12-11 11:12:55,575 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:12:55,575 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: get_summary +2024-12-11 11:12:55,575 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-11 11:12:55,576 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:55,652 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:12:55,652 INFO MainThread:736906 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-11 11:12:55,652 INFO MainThread:736906 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-11 11:12:55,653 INFO MainThread:736906 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-11 11:12:55,653 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: shutdown +2024-12-11 11:12:55,653 INFO HandlerThread:736906 [handler.py:finish():882] shutting down handler +2024-12-11 11:12:56,575 INFO WriterThread:736906 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/run-7k8jf8l5.wandb +2024-12-11 11:12:56,652 INFO SenderThread:736906 [sender.py:finish():1615] shutting down sender +2024-12-11 11:12:56,652 INFO SenderThread:736906 [file_pusher.py:finish():169] shutting down file pusher +2024-12-11 11:12:56,652 INFO SenderThread:736906 [file_pusher.py:join():175] waiting for file pusher diff --git a/110822/wandb/debug.log b/110822/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..dfce6deda856694095bb5922d4b33cde2714f342 --- /dev/null +++ b/110822/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-11 11:08:28,499 INFO MainThread:736760 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-11 11:08:28,499 INFO MainThread:736760 [wandb_setup.py:_flush():76] Configure stats pid to 736760 +2024-12-11 11:08:28,499 INFO MainThread:736760 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-11 11:08:28,499 INFO MainThread:736760 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/settings +2024-12-11 11:08:28,500 INFO MainThread:736760 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-11 11:08:28,500 INFO MainThread:736760 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-11 11:08:28,500 INFO MainThread:736760 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-11 11:08:28,500 INFO MainThread:736760 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-11 11:08:28,500 INFO MainThread:736760 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/logs/debug.log +2024-12-11 11:08:28,500 INFO MainThread:736760 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/logs/debug-internal.log +2024-12-11 11:08:28,500 INFO MainThread:736760 [wandb_init.py:init():569] calling init triggers +2024-12-11 11:08:28,500 INFO MainThread:736760 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '265644', 'wandb_notes': '265644_req_265752', 'eval': True, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim_left_to_right'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 16, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/265644', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'obs_shape': [5, 240, 320], 'action_shape': [36, 7], 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'true_action_history': False, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt'} +2024-12-11 11:08:28,500 INFO MainThread:736760 [wandb_init.py:init():619] starting backend +2024-12-11 11:08:28,500 INFO MainThread:736760 [wandb_init.py:init():623] setting up manager +2024-12-11 11:08:28,504 INFO MainThread:736760 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-11 11:08:28,505 INFO MainThread:736760 [wandb_init.py:init():631] backend started and connected +2024-12-11 11:08:28,525 INFO MainThread:736760 [wandb_init.py:init():720] updated telemetry +2024-12-11 11:08:28,531 INFO MainThread:736760 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-11 11:08:28,832 INFO MainThread:736760 [wandb_run.py:_on_init():2435] communicating current version +2024-12-11 11:08:28,875 INFO MainThread:736760 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.0 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-11 11:08:28,876 INFO MainThread:736760 [wandb_init.py:init():804] starting run threads in backend +2024-12-11 11:08:29,220 INFO MainThread:736760 [wandb_run.py:_console_start():2413] atexit reg +2024-12-11 11:08:29,224 INFO MainThread:736760 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-11 11:08:29,224 INFO MainThread:736760 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-11 11:08:29,225 INFO MainThread:736760 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-11 11:08:29,226 INFO MainThread:736760 [wandb_init.py:init():847] run started, returning control to user process +2024-12-11 11:08:29,227 INFO MainThread:736760 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/tb, True +2024-12-11 11:08:31,969 INFO MainThread:736760 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-11 11:12:56,656 WARNING MsgRouterThr:736760 [router.py:message_loop():77] message_loop has been closed diff --git a/110822/wandb/run-20241211_110828-7k8jf8l5/files/code/FISH/eval_robot.py b/110822/wandb/run-20241211_110828-7k8jf8l5/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..75a95b02db44bf1d6cdce4329109f52a0c4c1bc8 --- /dev/null +++ b/110822/wandb/run-20241211_110828-7k8jf8l5/files/code/FISH/eval_robot.py @@ -0,0 +1,512 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/110822/wandb/run-20241211_110828-7k8jf8l5/files/config.yaml b/110822/wandb/run-20241211_110828-7k8jf8l5/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..987d68cadd0541473420ea1779b091b4c2d62664 --- /dev/null +++ b/110822/wandb/run-20241211_110828-7k8jf8l5/files/config.yaml @@ -0,0 +1,966 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '265644' +wandb_notes: + desc: null + value: 265644_req_265752 +eval: + desc: null + value: true +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: + - hbm_twodim_left_to_right +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 16 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/265644 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + obs_shape: + - 5 + - 240 + - 320 + action_shape: + - 36 + - 7 + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + policy_frequency: 5 + interpolation_frequency: 25 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + policy_frequency: 5 + interpolation_frequency: 25 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +true_action_history: + desc: null + value: false +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1733933308 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 16 + - 23 + - 35 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/110822/wandb/run-20241211_110828-7k8jf8l5/files/diff.patch b/110822/wandb/run-20241211_110828-7k8jf8l5/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..aba9706539595845ca6018674c95f8b64a85c4fc --- /dev/null +++ b/110822/wandb/run-20241211_110828-7k8jf8l5/files/diff.patch @@ -0,0 +1,42 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 91f9f99..48bf485 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -124,18 +124,35 @@ bc_weight_type: 'qfilter' # linear, qfilter + # Load weights + load_checkpoint: ${agent.load_checkpoint} + # 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +-# full dataset ++# full 32 demos + # wandb_run_id: "265626" # RGBD+Mask+act history seed 0 + # wandb_run_id: "265638" # RGBD+Mask+act history seed 1 + # wandb_run_id: "265637" # RGBD+Mask+act history seed 2 + # wandb_run_id: "265636" # RGBD+Mask+act history seed 3 + # wandb_run_id: "265635" # RGBD+Mask+act history seed 4 + ++# 8 held out demos (26 demos) ++# wandb_run_id: "265652" # RGBD+Mask+contact+act history dataset seed 0 ++# wandb_run_id: "265654" # RGBD+Mask+contact+act history dataset seed 1 ++# wandb_run_id: "265655" # RGBD+Mask+contact+act history dataset seed 2 ++ ++# 8 held out demos (26 demos) ++# wandb_run_id: "265656" # RGBD+Mask+act history dataset seed 0 ++# wandb_run_id: "265657" # RGBD+Mask+act history dataset seed 1 ++# wandb_run_id: "265658" # RGBD+Mask+act history dataset seed 2 ++ ++# 16 held out demos (16 demos) ++# wandb_run_id: "265641" # RGBD+Mask+act history dataset seed 0 ++wandb_run_id: "265644" # RGBD+Mask+act history dataset seed 1 ++# wandb_run_id: "265645" # RGBD+Mask+act history dataset seed 2 ++# wandb_run_id: "266381_0" # RGBD+Mask+act history dataset seed 4 ++ + # wandb_run_id: "265628" # RGBD+Mask+contact+act history seed 0 + # wandb_run_id: "265630" # RGBD+Mask+contact+act history seed 1 + # wandb_run_id: "265631" # RGBD+Mask+contact+act history seed 2 + # wandb_run_id: "265633" # RGBD+Mask+contact+act history seed 3 +-wandb_run_id: "265634" # RGBD+Mask+contact+act history seed 4 ++# wandb_run_id: "265634" # RGBD+Mask+contact+act history seed 4 ++ + + # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # full dataset diff --git a/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/0_eval_0_a7ce057e6fe1a72a45a4.mp4 b/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/0_eval_0_a7ce057e6fe1a72a45a4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8c3fa49623844076f5e07b5b66b2d228e30bda00 --- /dev/null +++ b/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/0_eval_0_a7ce057e6fe1a72a45a4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7ce057e6fe1a72a45a4317170a65df82df6555562cdad1d7cd7243949470e1a +size 1148214 diff --git a/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/1_eval_1_dce37a039a0f3ae510e5.mp4 b/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/1_eval_1_dce37a039a0f3ae510e5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df434b685b801cad39682ce3553022a610ec95f1 --- /dev/null +++ b/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/1_eval_1_dce37a039a0f3ae510e5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dce37a039a0f3ae510e5dcc44fee8b8ef5b47c9b606de8657c821538ad6b0900 +size 1152733 diff --git a/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/2_eval_2_c7cd658cf3e0e8f2dada.mp4 b/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/2_eval_2_c7cd658cf3e0e8f2dada.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ac3e90e1bbd9c99da70cf946e433a873ac4a54d1 --- /dev/null +++ b/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/2_eval_2_c7cd658cf3e0e8f2dada.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7cd658cf3e0e8f2dada2bbb69769df6b96cb1af52105e6b852a5aa6e7dce071 +size 1195653 diff --git a/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/3_eval_3_ac712fc970ab6ee59b1c.mp4 b/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/3_eval_3_ac712fc970ab6ee59b1c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3aa5b9dc182b9d9b39e55098744684408ca0c7a8 --- /dev/null +++ b/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/3_eval_3_ac712fc970ab6ee59b1c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac712fc970ab6ee59b1c4c0c66fe4bb4b0ecb93b156e194c41d4bf46045d2870 +size 1242757 diff --git a/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/4_eval_4_b9f6613e7cd6916b7519.mp4 b/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/4_eval_4_b9f6613e7cd6916b7519.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d17c4d7e83ed371aa916136ad4a7e5b1a3e6ba4b --- /dev/null +++ b/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/4_eval_4_b9f6613e7cd6916b7519.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9f6613e7cd6916b75191e5752ebdf1ff8273ec0404a1b2cc3197bf4e1dbcc81 +size 1261098 diff --git a/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log b/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..b9514274cf35598a2e644aaca53549f75ec3b5e5 --- /dev/null +++ b/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log @@ -0,0 +1,6330 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1733933311.970296]: resetting environment +[INFO] [1733933311.979890]: cleared current plan +[INFO] [1733933311.981144]: moving to home +[INFO] [1733933312.983707]: reached home +[INFO] [1733933312.984996]: reset action history +[INFO] [1733933315.692938]: environment reset +Starting episode 0 +[INFO] [1733933315.694787]: resetting environment +[INFO] [1733933315.697998]: cleared current plan +[INFO] [1733933315.698780]: moving to home +[INFO] [1733933316.701745]: reached home +[INFO] [1733933316.703333]: reset action history +[INFO] [1733933319.411012]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1733933323.153867]: resetting environment +[INFO] [1733933323.156844]: cleared current plan +[INFO] [1733933323.157932]: moving to home +[INFO] [1733933324.160526]: reached home +[INFO] [1733933324.162543]: reset action history +[INFO] [1733933326.870230]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/episode_rosbags/episode_0_2024-12-11-11-08-46.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-736760.log +started roslaunch server http://158.130.50.37:45317/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [737007] +process[print_text-2]: started with pid [737008] +process[rosbag_record-3]: started with pid [737031] +started bagging! +[INFO] [1733933327.650352]: received mask_obs with header seq: 18734 +stamp: + secs: 1733933327 + nsecs: 651036977 +frame_id: '' and grasped_object_mask header seq: 450296 +stamp: + secs: 1733933327 + nsecs: 411805630 +frame_id: "camera_color_optical_frame" +[INFO] [1733933327.830407]: received mask_obs with header seq: 18735 +stamp: + secs: 1733933327 + nsecs: 830882310 +frame_id: '' and grasped_object_mask header seq: 450302 +stamp: + secs: 1733933327 + nsecs: 611642838 +frame_id: "camera_color_optical_frame" +[INFO] [1733933328.212968]: received mask_obs with header seq: 18736 +stamp: + secs: 1733933328 + nsecs: 213523626 +frame_id: '' and grasped_object_mask header seq: 450314 +stamp: + secs: 1733933328 + nsecs: 11320829 +frame_id: "camera_color_optical_frame" +[INFO] [1733933328.383139]: received mask_obs with header seq: 18737 +stamp: + secs: 1733933328 + nsecs: 384063243 +frame_id: '' and grasped_object_mask header seq: 450319 +stamp: + secs: 1733933328 + nsecs: 177854538 +frame_id: "camera_color_optical_frame" +[INFO] [1733933328.570704]: received mask_obs with header seq: 18738 +stamp: + secs: 1733933328 + nsecs: 571686983 +frame_id: '' and grasped_object_mask header seq: 450324 +stamp: + secs: 1733933328 + nsecs: 344387054 +frame_id: "camera_color_optical_frame" +[INFO] [1733933328.756583]: received mask_obs with header seq: 18739 +stamp: + secs: 1733933328 + nsecs: 757521390 +frame_id: '' and grasped_object_mask header seq: 450330 +stamp: + secs: 1733933328 + nsecs: 544226885 +frame_id: "camera_color_optical_frame" +[INFO] [1733933328.958541]: received mask_obs with header seq: 18740 +stamp: + secs: 1733933328 + nsecs: 959535837 +frame_id: '' and grasped_object_mask header seq: 450336 +stamp: + secs: 1733933328 + nsecs: 744068146 +frame_id: "camera_color_optical_frame" +[INFO] [1733933329.146424]: received mask_obs with header seq: 18741 +stamp: + secs: 1733933329 + nsecs: 147156953 +frame_id: '' and grasped_object_mask header seq: 450341 +stamp: + secs: 1733933328 + nsecs: 910602093 +frame_id: "camera_color_optical_frame" +[INFO] [1733933329.343714]: received mask_obs with header seq: 18742 +stamp: + secs: 1733933329 + nsecs: 344312906 +frame_id: '' and grasped_object_mask header seq: 450347 +stamp: + secs: 1733933329 + nsecs: 110443592 +frame_id: "camera_color_optical_frame" +[INFO] [1733933329.537245]: received mask_obs with header seq: 18743 +stamp: + secs: 1733933329 + nsecs: 538162469 +frame_id: '' and grasped_object_mask header seq: 450353 +stamp: + secs: 1733933329 + nsecs: 310286045 +frame_id: "camera_color_optical_frame" +[INFO] [1733933329.737001]: received mask_obs with header seq: 18744 +stamp: + secs: 1733933329 + nsecs: 737591981 +frame_id: '' and grasped_object_mask header seq: 450359 +stamp: + secs: 1733933329 + nsecs: 510128260 +frame_id: "camera_color_optical_frame" +[INFO] [1733933329.928733]: received mask_obs with header seq: 18745 +stamp: + secs: 1733933329 + nsecs: 929746866 +frame_id: '' and grasped_object_mask header seq: 450365 +stamp: + secs: 1733933329 + nsecs: 709969044 +frame_id: "camera_color_optical_frame" +[INFO] [1733933330.129441]: received mask_obs with header seq: 18746 +stamp: + secs: 1733933330 + nsecs: 128825426 +frame_id: '' and grasped_object_mask header seq: 450371 +stamp: + secs: 1733933329 + nsecs: 909803152 +frame_id: "camera_color_optical_frame" +[INFO] [1733933330.327553]: received mask_obs with header seq: 18747 +stamp: + secs: 1733933330 + nsecs: 328436851 +frame_id: '' and grasped_object_mask header seq: 450377 +stamp: + secs: 1733933330 + nsecs: 109637260 +frame_id: "camera_color_optical_frame" +[INFO] [1733933330.526266]: received mask_obs with header seq: 18748 +stamp: + secs: 1733933330 + nsecs: 526973009 +frame_id: '' and grasped_object_mask header seq: 450383 +stamp: + secs: 1733933330 + nsecs: 309470892 +frame_id: "camera_color_optical_frame" +[INFO] [1733933330.720478]: received mask_obs with header seq: 18749 +stamp: + secs: 1733933330 + nsecs: 721301078 +frame_id: '' and grasped_object_mask header seq: 450388 +stamp: + secs: 1733933330 + nsecs: 475999594 +frame_id: "camera_color_optical_frame" +[INFO] [1733933330.913197]: received mask_obs with header seq: 18750 +stamp: + secs: 1733933330 + nsecs: 914164543 +frame_id: '' and grasped_object_mask header seq: 450394 +stamp: + secs: 1733933330 + nsecs: 675832748 +frame_id: "camera_color_optical_frame" +[INFO] [1733933331.109388]: received mask_obs with header seq: 18751 +stamp: + secs: 1733933331 + nsecs: 110332489 +frame_id: '' and grasped_object_mask header seq: 450400 +stamp: + secs: 1733933330 + nsecs: 875665426 +frame_id: "camera_color_optical_frame" +[INFO] [1733933331.309965]: received mask_obs with header seq: 18752 +stamp: + secs: 1733933331 + nsecs: 310950994 +frame_id: '' and grasped_object_mask header seq: 450406 +stamp: + secs: 1733933331 + nsecs: 75497866 +frame_id: "camera_color_optical_frame" +[INFO] [1733933331.491686]: received mask_obs with header seq: 18753 +stamp: + secs: 1733933331 + nsecs: 492656469 +frame_id: '' and grasped_object_mask header seq: 450411 +stamp: + secs: 1733933331 + nsecs: 242025137 +frame_id: "camera_color_optical_frame" +[INFO] [1733933331.687860]: received mask_obs with header seq: 18754 +stamp: + secs: 1733933331 + nsecs: 688904523 +frame_id: '' and grasped_object_mask header seq: 450417 +stamp: + secs: 1733933331 + nsecs: 441857100 +frame_id: "camera_color_optical_frame" +[INFO] [1733933331.882596]: received mask_obs with header seq: 18755 +stamp: + secs: 1733933331 + nsecs: 883145570 +frame_id: '' and grasped_object_mask header seq: 450423 +stamp: + secs: 1733933331 + nsecs: 641689062 +frame_id: "camera_color_optical_frame" +[INFO] [1733933332.074137]: received mask_obs with header seq: 18756 +stamp: + secs: 1733933332 + nsecs: 75003623 +frame_id: '' and grasped_object_mask header seq: 450429 +stamp: + secs: 1733933331 + nsecs: 841527700 +frame_id: "camera_color_optical_frame" +[INFO] [1733933332.266314]: received mask_obs with header seq: 18757 +stamp: + secs: 1733933332 + nsecs: 267266035 +frame_id: '' and grasped_object_mask header seq: 450434 +stamp: + secs: 1733933332 + nsecs: 8059740 +frame_id: "camera_color_optical_frame" +[INFO] [1733933332.452433]: received mask_obs with header seq: 18758 +stamp: + secs: 1733933332 + nsecs: 453300952 +frame_id: '' and grasped_object_mask header seq: 450441 +stamp: + secs: 1733933332 + nsecs: 241206646 +frame_id: "camera_color_optical_frame" +[INFO] [1733933332.642957]: received mask_obs with header seq: 18759 +stamp: + secs: 1733933332 + nsecs: 643815517 +frame_id: '' and grasped_object_mask header seq: 450446 +stamp: + secs: 1733933332 + nsecs: 407740116 +frame_id: "camera_color_optical_frame" +[INFO] [1733933332.838953]: received mask_obs with header seq: 18760 +stamp: + secs: 1733933332 + nsecs: 839906454 +frame_id: '' and grasped_object_mask header seq: 450452 +stamp: + secs: 1733933332 + nsecs: 607580185 +frame_id: "camera_color_optical_frame" +[INFO] [1733933333.025038]: received mask_obs with header seq: 18761 +stamp: + secs: 1733933333 + nsecs: 25934219 +frame_id: '' and grasped_object_mask header seq: 450457 +stamp: + secs: 1733933332 + nsecs: 774110079 +frame_id: "camera_color_optical_frame" +[INFO] [1733933333.228004]: received mask_obs with header seq: 18762 +stamp: + secs: 1733933333 + nsecs: 228936672 +frame_id: '' and grasped_object_mask header seq: 450464 +stamp: + secs: 1733933333 + nsecs: 7250071 +frame_id: "camera_color_optical_frame" +[INFO] [1733933333.425212]: received mask_obs with header seq: 18763 +stamp: + secs: 1733933333 + nsecs: 426135540 +frame_id: '' and grasped_object_mask header seq: 450470 +stamp: + secs: 1733933333 + nsecs: 207083225 +frame_id: "camera_color_optical_frame" +[INFO] [1733933333.624668]: received mask_obs with header seq: 18764 +stamp: + secs: 1733933333 + nsecs: 625190258 +frame_id: '' and grasped_object_mask header seq: 450476 +stamp: + secs: 1733933333 + nsecs: 406916618 +frame_id: "camera_color_optical_frame" +[INFO] [1733933333.828912]: received mask_obs with header seq: 18765 +stamp: + secs: 1733933333 + nsecs: 828082323 +frame_id: '' and grasped_object_mask header seq: 450482 +stamp: + secs: 1733933333 + nsecs: 606750488 +frame_id: "camera_color_optical_frame" +[INFO] [1733933334.024892]: received mask_obs with header seq: 18766 +stamp: + secs: 1733933334 + nsecs: 25610208 +frame_id: '' and grasped_object_mask header seq: 450488 +stamp: + secs: 1733933333 + nsecs: 806586504 +frame_id: "camera_color_optical_frame" +[INFO] [1733933334.228357]: received mask_obs with header seq: 18767 +stamp: + secs: 1733933334 + nsecs: 228415727 +frame_id: '' and grasped_object_mask header seq: 450494 +stamp: + secs: 1733933334 + nsecs: 6424189 +frame_id: "camera_color_optical_frame" +[INFO] [1733933334.424166]: received mask_obs with header seq: 18768 +stamp: + secs: 1733933334 + nsecs: 424822807 +frame_id: '' and grasped_object_mask header seq: 450500 +stamp: + secs: 1733933334 + nsecs: 206261873 +frame_id: "camera_color_optical_frame" +[INFO] [1733933334.624736]: received mask_obs with header seq: 18769 +stamp: + secs: 1733933334 + nsecs: 625626325 +frame_id: '' and grasped_object_mask header seq: 450506 +stamp: + secs: 1733933334 + nsecs: 406099796 +frame_id: "camera_color_optical_frame" +[INFO] [1733933334.823772]: received mask_obs with header seq: 18770 +stamp: + secs: 1733933334 + nsecs: 824816226 +frame_id: '' and grasped_object_mask header seq: 450512 +stamp: + secs: 1733933334 + nsecs: 605937958 +frame_id: "camera_color_optical_frame" +[INFO] [1733933335.025815]: received mask_obs with header seq: 18771 +stamp: + secs: 1733933335 + nsecs: 26367187 +frame_id: '' and grasped_object_mask header seq: 450518 +stamp: + secs: 1733933334 + nsecs: 805773258 +frame_id: "camera_color_optical_frame" +[INFO] [1733933335.227538]: received mask_obs with header seq: 18772 +stamp: + secs: 1733933335 + nsecs: 227639198 +frame_id: '' and grasped_object_mask header seq: 450524 +stamp: + secs: 1733933335 + nsecs: 5607128 +frame_id: "camera_color_optical_frame" +[INFO] [1733933335.427139]: received mask_obs with header seq: 18773 +stamp: + secs: 1733933335 + nsecs: 426608085 +frame_id: '' and grasped_object_mask header seq: 450530 +stamp: + secs: 1733933335 + nsecs: 205440283 +frame_id: "camera_color_optical_frame" +[INFO] [1733933335.624206]: received mask_obs with header seq: 18774 +stamp: + secs: 1733933335 + nsecs: 625132322 +frame_id: '' and grasped_object_mask header seq: 450536 +stamp: + secs: 1733933335 + nsecs: 405272961 +frame_id: "camera_color_optical_frame" +[INFO] [1733933335.824383]: received mask_obs with header seq: 18775 +stamp: + secs: 1733933335 + nsecs: 825085163 +frame_id: '' and grasped_object_mask header seq: 450542 +stamp: + secs: 1733933335 + nsecs: 605105639 +frame_id: "camera_color_optical_frame" +[INFO] [1733933336.025471]: received mask_obs with header seq: 18776 +stamp: + secs: 1733933336 + nsecs: 25911808 +frame_id: '' and grasped_object_mask header seq: 450548 +stamp: + secs: 1733933335 + nsecs: 804938316 +frame_id: "camera_color_optical_frame" +[INFO] [1733933336.235139]: received mask_obs with header seq: 18777 +stamp: + secs: 1733933336 + nsecs: 236074924 +frame_id: '' and grasped_object_mask header seq: 450554 +stamp: + secs: 1733933336 + nsecs: 4770517 +frame_id: "camera_color_optical_frame" +[INFO] [1733933336.433255]: received mask_obs with header seq: 18778 +stamp: + secs: 1733933336 + nsecs: 434269189 +frame_id: '' and grasped_object_mask header seq: 450560 +stamp: + secs: 1733933336 + nsecs: 204603434 +frame_id: "camera_color_optical_frame" +[INFO] [1733933336.623981]: received mask_obs with header seq: 18779 +stamp: + secs: 1733933336 + nsecs: 624917507 +frame_id: '' and grasped_object_mask header seq: 450566 +stamp: + secs: 1733933336 + nsecs: 404435158 +frame_id: "camera_color_optical_frame" +[INFO] [1733933336.825388]: received mask_obs with header seq: 18780 +stamp: + secs: 1733933336 + nsecs: 825251817 +frame_id: '' and grasped_object_mask header seq: 450572 +stamp: + secs: 1733933336 + nsecs: 604266405 +frame_id: "camera_color_optical_frame" +[INFO] [1733933337.052189]: received mask_obs with header seq: 18781 +stamp: + secs: 1733933337 + nsecs: 53012371 +frame_id: '' and grasped_object_mask header seq: 450578 +stamp: + secs: 1733933336 + nsecs: 804100990 +frame_id: "camera_color_optical_frame" +[INFO] [1733933337.239423]: received mask_obs with header seq: 18782 +stamp: + secs: 1733933337 + nsecs: 240421295 +frame_id: '' and grasped_object_mask header seq: 450584 +stamp: + secs: 1733933337 + nsecs: 3937960 +frame_id: "camera_color_optical_frame" +[INFO] [1733933337.431373]: received mask_obs with header seq: 18783 +stamp: + secs: 1733933337 + nsecs: 431360721 +frame_id: '' and grasped_object_mask header seq: 450590 +stamp: + secs: 1733933337 + nsecs: 203774929 +frame_id: "camera_color_optical_frame" +[INFO] [1733933337.626604]: received mask_obs with header seq: 18784 +stamp: + secs: 1733933337 + nsecs: 625090360 +frame_id: '' and grasped_object_mask header seq: 450596 +stamp: + secs: 1733933337 + nsecs: 403613091 +frame_id: "camera_color_optical_frame" +[INFO] [1733933337.826348]: received mask_obs with header seq: 18785 +stamp: + secs: 1733933337 + nsecs: 825843095 +frame_id: '' and grasped_object_mask header seq: 450602 +stamp: + secs: 1733933337 + nsecs: 603449821 +frame_id: "camera_color_optical_frame" +[INFO] [1733933338.026924]: received mask_obs with header seq: 18786 +stamp: + secs: 1733933338 + nsecs: 27883291 +frame_id: '' and grasped_object_mask header seq: 450608 +stamp: + secs: 1733933337 + nsecs: 803287506 +frame_id: "camera_color_optical_frame" +[INFO] [1733933338.229337]: received mask_obs with header seq: 18787 +stamp: + secs: 1733933338 + nsecs: 228755474 +frame_id: '' and grasped_object_mask header seq: 450614 +stamp: + secs: 1733933338 + nsecs: 3125906 +frame_id: "camera_color_optical_frame" +[INFO] [1733933338.424058]: received mask_obs with header seq: 18788 +stamp: + secs: 1733933338 + nsecs: 424803495 +frame_id: '' and grasped_object_mask header seq: 450620 +stamp: + secs: 1733933338 + nsecs: 202964306 +frame_id: "camera_color_optical_frame" +[INFO] [1733933338.635809]: received mask_obs with header seq: 18789 +stamp: + secs: 1733933338 + nsecs: 624933958 +frame_id: '' and grasped_object_mask header seq: 450626 +stamp: + secs: 1733933338 + nsecs: 402802706 +frame_id: "camera_color_optical_frame" +[INFO] [1733933338.823138]: received mask_obs with header seq: 18790 +stamp: + secs: 1733933338 + nsecs: 824217319 +frame_id: '' and grasped_object_mask header seq: 450632 +stamp: + secs: 1733933338 + nsecs: 602642298 +frame_id: "camera_color_optical_frame" +[INFO] [1733933339.023802]: received mask_obs with header seq: 18791 +stamp: + secs: 1733933339 + nsecs: 24708509 +frame_id: '' and grasped_object_mask header seq: 450638 +stamp: + secs: 1733933338 + nsecs: 802479982 +frame_id: "camera_color_optical_frame" +[INFO] [1733933339.230853]: received mask_obs with header seq: 18792 +stamp: + secs: 1733933339 + nsecs: 231801033 +frame_id: '' and grasped_object_mask header seq: 450644 +stamp: + secs: 1733933339 + nsecs: 2317667 +frame_id: "camera_color_optical_frame" +[INFO] [1733933339.425390]: received mask_obs with header seq: 18793 +stamp: + secs: 1733933339 + nsecs: 425042390 +frame_id: '' and grasped_object_mask header seq: 450650 +stamp: + secs: 1733933339 + nsecs: 202155113 +frame_id: "camera_color_optical_frame" +[INFO] [1733933339.628628]: received mask_obs with header seq: 18794 +stamp: + secs: 1733933339 + nsecs: 629562854 +frame_id: '' and grasped_object_mask header seq: 450656 +stamp: + secs: 1733933339 + nsecs: 401992559 +frame_id: "camera_color_optical_frame" +[INFO] [1733933339.826185]: received mask_obs with header seq: 18795 +stamp: + secs: 1733933339 + nsecs: 827060937 +frame_id: '' and grasped_object_mask header seq: 450662 +stamp: + secs: 1733933339 + nsecs: 601830244 +frame_id: "camera_color_optical_frame" +[INFO] [1733933340.028662]: received mask_obs with header seq: 18796 +stamp: + secs: 1733933340 + nsecs: 29277086 +frame_id: '' and grasped_object_mask header seq: 450668 +stamp: + secs: 1733933339 + nsecs: 801671028 +frame_id: "camera_color_optical_frame" +[INFO] [1733933340.231841]: received mask_obs with header seq: 18797 +stamp: + secs: 1733933340 + nsecs: 232812404 +frame_id: '' and grasped_object_mask header seq: 450674 +stamp: + secs: 1733933340 + nsecs: 1512766 +frame_id: "camera_color_optical_frame" +[INFO] [1733933340.425719]: received mask_obs with header seq: 18798 +stamp: + secs: 1733933340 + nsecs: 424967050 +frame_id: '' and grasped_object_mask header seq: 450680 +stamp: + secs: 1733933340 + nsecs: 201354265 +frame_id: "camera_color_optical_frame" +[INFO] [1733933340.626085]: received mask_obs with header seq: 18799 +stamp: + secs: 1733933340 + nsecs: 626764535 +frame_id: '' and grasped_object_mask header seq: 450686 +stamp: + secs: 1733933340 + nsecs: 401196003 +frame_id: "camera_color_optical_frame" +[INFO] [1733933340.826623]: received mask_obs with header seq: 18800 +stamp: + secs: 1733933340 + nsecs: 827700138 +frame_id: '' and grasped_object_mask header seq: 450692 +stamp: + secs: 1733933340 + nsecs: 601037979 +frame_id: "camera_color_optical_frame" +[INFO] [1733933341.025919]: received mask_obs with header seq: 18801 +stamp: + secs: 1733933341 + nsecs: 26716947 +frame_id: '' and grasped_object_mask header seq: 450698 +stamp: + secs: 1733933340 + nsecs: 800880194 +frame_id: "camera_color_optical_frame" +[INFO] [1733933341.232368]: received mask_obs with header seq: 18802 +stamp: + secs: 1733933341 + nsecs: 233354806 +frame_id: '' and grasped_object_mask header seq: 450704 +stamp: + secs: 1733933341 + nsecs: 723124 +frame_id: "camera_color_optical_frame" +[INFO] [1733933341.441987]: received mask_obs with header seq: 18803 +stamp: + secs: 1733933341 + nsecs: 442978382 +frame_id: '' and grasped_object_mask header seq: 450710 +stamp: + secs: 1733933341 + nsecs: 200565100 +frame_id: "camera_color_optical_frame" +[INFO] [1733933341.633553]: received mask_obs with header seq: 18804 +stamp: + secs: 1733933341 + nsecs: 633615255 +frame_id: '' and grasped_object_mask header seq: 450716 +stamp: + secs: 1733933341 + nsecs: 400407553 +frame_id: "camera_color_optical_frame" +[INFO] [1733933341.824052]: received mask_obs with header seq: 18805 +stamp: + secs: 1733933341 + nsecs: 824878692 +frame_id: '' and grasped_object_mask header seq: 450722 +stamp: + secs: 1733933341 + nsecs: 600250006 +frame_id: "camera_color_optical_frame" +[INFO] [1733933342.025374]: received mask_obs with header seq: 18806 +stamp: + secs: 1733933342 + nsecs: 24950504 +frame_id: '' and grasped_object_mask header seq: 450728 +stamp: + secs: 1733933341 + nsecs: 800092936 +frame_id: "camera_color_optical_frame" +[INFO] [1733933342.229859]: received mask_obs with header seq: 18807 +stamp: + secs: 1733933342 + nsecs: 230894327 +frame_id: '' and grasped_object_mask header seq: 450734 +stamp: + secs: 1733933341 + nsecs: 999936581 +frame_id: "camera_color_optical_frame" +[INFO] [1733933342.425112]: received mask_obs with header seq: 18808 +stamp: + secs: 1733933342 + nsecs: 424909353 +frame_id: '' and grasped_object_mask header seq: 450740 +stamp: + secs: 1733933342 + nsecs: 199780226 +frame_id: "camera_color_optical_frame" +[INFO] [1733933342.633170]: received mask_obs with header seq: 18809 +stamp: + secs: 1733933342 + nsecs: 633015155 +frame_id: '' and grasped_object_mask header seq: 450746 +stamp: + secs: 1733933342 + nsecs: 399625540 +frame_id: "camera_color_optical_frame" +[INFO] [1733933342.825253]: received mask_obs with header seq: 18810 +stamp: + secs: 1733933342 + nsecs: 826218128 +frame_id: '' and grasped_object_mask header seq: 450752 +stamp: + secs: 1733933342 + nsecs: 599469662 +frame_id: "camera_color_optical_frame" +[INFO] [1733933343.024968]: received mask_obs with header seq: 18811 +stamp: + secs: 1733933343 + nsecs: 24919033 +frame_id: '' and grasped_object_mask header seq: 450758 +stamp: + secs: 1733933342 + nsecs: 799310923 +frame_id: "camera_color_optical_frame" +[INFO] [1733933343.227679]: received mask_obs with header seq: 18812 +stamp: + secs: 1733933343 + nsecs: 228591203 +frame_id: '' and grasped_object_mask header seq: 450764 +stamp: + secs: 1733933342 + nsecs: 999150276 +frame_id: "camera_color_optical_frame" +[INFO] [1733933343.424920]: received mask_obs with header seq: 18813 +stamp: + secs: 1733933343 + nsecs: 425454616 +frame_id: '' and grasped_object_mask header seq: 450770 +stamp: + secs: 1733933343 + nsecs: 198989630 +frame_id: "camera_color_optical_frame" +[INFO] [1733933343.625800]: received mask_obs with header seq: 18814 +stamp: + secs: 1733933343 + nsecs: 625307559 +frame_id: '' and grasped_object_mask header seq: 450776 +stamp: + secs: 1733933343 + nsecs: 398829460 +frame_id: "camera_color_optical_frame" +[INFO] [1733933343.822881]: received mask_obs with header seq: 18815 +stamp: + secs: 1733933343 + nsecs: 823783159 +frame_id: '' and grasped_object_mask header seq: 450782 +stamp: + secs: 1733933343 + nsecs: 598670006 +frame_id: "camera_color_optical_frame" +[INFO] [1733933344.025037]: received mask_obs with header seq: 18816 +stamp: + secs: 1733933344 + nsecs: 24611234 +frame_id: '' and grasped_object_mask header seq: 450788 +stamp: + secs: 1733933343 + nsecs: 798512220 +frame_id: "camera_color_optical_frame" +[INFO] [1733933344.224894]: received mask_obs with header seq: 18817 +stamp: + secs: 1733933344 + nsecs: 225749969 +frame_id: '' and grasped_object_mask header seq: 450794 +stamp: + secs: 1733933343 + nsecs: 998356342 +frame_id: "camera_color_optical_frame" +[INFO] [1733933344.428017]: received mask_obs with header seq: 18818 +stamp: + secs: 1733933344 + nsecs: 428788900 +frame_id: '' and grasped_object_mask header seq: 450800 +stamp: + secs: 1733933344 + nsecs: 198201180 +frame_id: "camera_color_optical_frame" +[INFO] [1733933344.624590]: received mask_obs with header seq: 18819 +stamp: + secs: 1733933344 + nsecs: 623894214 +frame_id: '' and grasped_object_mask header seq: 450806 +stamp: + secs: 1733933344 + nsecs: 398046255 +frame_id: "camera_color_optical_frame" +[INFO] [1733933344.827045]: received mask_obs with header seq: 18820 +stamp: + secs: 1733933344 + nsecs: 828097343 +frame_id: '' and grasped_object_mask header seq: 450812 +stamp: + secs: 1733933344 + nsecs: 597891331 +frame_id: "camera_color_optical_frame" +[INFO] [1733933345.026404]: received mask_obs with header seq: 18821 +stamp: + secs: 1733933345 + nsecs: 25313377 +frame_id: '' and grasped_object_mask header seq: 450818 +stamp: + secs: 1733933344 + nsecs: 797737122 +frame_id: "camera_color_optical_frame" +[INFO] [1733933345.228749]: received mask_obs with header seq: 18822 +stamp: + secs: 1733933345 + nsecs: 228934764 +frame_id: '' and grasped_object_mask header seq: 450824 +stamp: + secs: 1733933344 + nsecs: 997581005 +frame_id: "camera_color_optical_frame" +[INFO] [1733933345.427126]: received mask_obs with header seq: 18823 +stamp: + secs: 1733933345 + nsecs: 427857398 +frame_id: '' and grasped_object_mask header seq: 450830 +stamp: + secs: 1733933345 + nsecs: 197425604 +frame_id: "camera_color_optical_frame" +[INFO] [1733933345.628880]: received mask_obs with header seq: 18824 +stamp: + secs: 1733933345 + nsecs: 629968881 +frame_id: '' and grasped_object_mask header seq: 450836 +stamp: + secs: 1733933345 + nsecs: 397270441 +frame_id: "camera_color_optical_frame" +[INFO] [1733933345.824886]: received mask_obs with header seq: 18825 +stamp: + secs: 1733933345 + nsecs: 825074195 +frame_id: '' and grasped_object_mask header seq: 450842 +stamp: + secs: 1733933345 + nsecs: 597115517 +frame_id: "camera_color_optical_frame" +[INFO] [1733933346.026126]: received mask_obs with header seq: 18826 +stamp: + secs: 1733933346 + nsecs: 27090787 +frame_id: '' and grasped_object_mask header seq: 450848 +stamp: + secs: 1733933345 + nsecs: 796961069 +frame_id: "camera_color_optical_frame" +[INFO] [1733933346.229587]: received mask_obs with header seq: 18827 +stamp: + secs: 1733933346 + nsecs: 230442523 +frame_id: '' and grasped_object_mask header seq: 450854 +stamp: + secs: 1733933345 + nsecs: 996809244 +frame_id: "camera_color_optical_frame" +[INFO] [1733933346.428596]: received mask_obs with header seq: 18828 +stamp: + secs: 1733933346 + nsecs: 429663419 +frame_id: '' and grasped_object_mask header seq: 450860 +stamp: + secs: 1733933346 + nsecs: 196656466 +frame_id: "camera_color_optical_frame" +[INFO] [1733933346.625636]: received mask_obs with header seq: 18829 +stamp: + secs: 1733933346 + nsecs: 625269651 +frame_id: '' and grasped_object_mask header seq: 450866 +stamp: + secs: 1733933346 + nsecs: 396504164 +frame_id: "camera_color_optical_frame" +[INFO] [1733933346.825899]: received mask_obs with header seq: 18830 +stamp: + secs: 1733933346 + nsecs: 826886892 +frame_id: '' and grasped_object_mask header seq: 450872 +stamp: + secs: 1733933346 + nsecs: 596352339 +frame_id: "camera_color_optical_frame" +[INFO] [1733933347.020192]: received mask_obs with header seq: 18831 +stamp: + secs: 1733933347 + nsecs: 20363092 +frame_id: '' and grasped_object_mask header seq: 450878 +stamp: + secs: 1733933346 + nsecs: 796196699 +frame_id: "camera_color_optical_frame" +[INFO] [1733933347.223921]: received mask_obs with header seq: 18832 +stamp: + secs: 1733933347 + nsecs: 224977731 +frame_id: '' and grasped_object_mask header seq: 450884 +stamp: + secs: 1733933346 + nsecs: 996037483 +frame_id: "camera_color_optical_frame" +[INFO] [1733933347.419064]: received mask_obs with header seq: 18833 +stamp: + secs: 1733933347 + nsecs: 420123100 +frame_id: '' and grasped_object_mask header seq: 450890 +stamp: + secs: 1733933347 + nsecs: 195879698 +frame_id: "camera_color_optical_frame" +[INFO] [1733933347.620266]: received mask_obs with header seq: 18834 +stamp: + secs: 1733933347 + nsecs: 621035337 +frame_id: '' and grasped_object_mask header seq: 450896 +stamp: + secs: 1733933347 + nsecs: 395720482 +frame_id: "camera_color_optical_frame" +[INFO] [1733933347.824111]: received mask_obs with header seq: 18835 +stamp: + secs: 1733933347 + nsecs: 825014591 +frame_id: '' and grasped_object_mask header seq: 450902 +stamp: + secs: 1733933347 + nsecs: 595561266 +frame_id: "camera_color_optical_frame" +[INFO] [1733933348.023617]: received mask_obs with header seq: 18836 +stamp: + secs: 1733933348 + nsecs: 24389743 +frame_id: '' and grasped_object_mask header seq: 450908 +stamp: + secs: 1733933347 + nsecs: 795403004 +frame_id: "camera_color_optical_frame" +[INFO] [1733933348.222199]: received mask_obs with header seq: 18837 +stamp: + secs: 1733933348 + nsecs: 223239898 +frame_id: '' and grasped_object_mask header seq: 450914 +stamp: + secs: 1733933347 + nsecs: 995245934 +frame_id: "camera_color_optical_frame" +[INFO] [1733933348.420807]: received mask_obs with header seq: 18838 +stamp: + secs: 1733933348 + nsecs: 420935869 +frame_id: '' and grasped_object_mask header seq: 450920 +stamp: + secs: 1733933348 + nsecs: 195088625 +frame_id: "camera_color_optical_frame" +[INFO] [1733933348.624720]: received mask_obs with header seq: 18839 +stamp: + secs: 1733933348 + nsecs: 625223398 +frame_id: '' and grasped_object_mask header seq: 450926 +stamp: + secs: 1733933348 + nsecs: 394932508 +frame_id: "camera_color_optical_frame" +[INFO] [1733933348.818794]: received mask_obs with header seq: 18840 +stamp: + secs: 1733933348 + nsecs: 819845438 +frame_id: '' and grasped_object_mask header seq: 450932 +stamp: + secs: 1733933348 + nsecs: 594775915 +frame_id: "camera_color_optical_frame" +[INFO] [1733933349.021868]: received mask_obs with header seq: 18841 +stamp: + secs: 1733933349 + nsecs: 22546768 +frame_id: '' and grasped_object_mask header seq: 450938 +stamp: + secs: 1733933348 + nsecs: 794617176 +frame_id: "camera_color_optical_frame" +[INFO] [1733933349.231172]: received mask_obs with header seq: 18842 +stamp: + secs: 1733933349 + nsecs: 232270240 +frame_id: '' and grasped_object_mask header seq: 450944 +stamp: + secs: 1733933348 + nsecs: 994456291 +frame_id: "camera_color_optical_frame" +[INFO] [1733933349.430513]: received mask_obs with header seq: 18843 +stamp: + secs: 1733933349 + nsecs: 431483745 +frame_id: '' and grasped_object_mask header seq: 450950 +stamp: + secs: 1733933349 + nsecs: 194295645 +frame_id: "camera_color_optical_frame" +[INFO] [1733933349.626377]: received mask_obs with header seq: 18844 +stamp: + secs: 1733933349 + nsecs: 627464771 +frame_id: '' and grasped_object_mask header seq: 450956 +stamp: + secs: 1733933349 + nsecs: 394134760 +frame_id: "camera_color_optical_frame" +[INFO] [1733933349.822615]: received mask_obs with header seq: 18845 +stamp: + secs: 1733933349 + nsecs: 823675870 +frame_id: '' and grasped_object_mask header seq: 450962 +stamp: + secs: 1733933349 + nsecs: 593974113 +frame_id: "camera_color_optical_frame" +[INFO] [1733933350.021224]: received mask_obs with header seq: 18846 +stamp: + secs: 1733933350 + nsecs: 22146463 +frame_id: '' and grasped_object_mask header seq: 450968 +stamp: + secs: 1733933349 + nsecs: 793811083 +frame_id: "camera_color_optical_frame" +[INFO] [1733933350.233179]: received mask_obs with header seq: 18847 +stamp: + secs: 1733933350 + nsecs: 234192848 +frame_id: '' and grasped_object_mask header seq: 450974 +stamp: + secs: 1733933349 + nsecs: 993643761 +frame_id: "camera_color_optical_frame" +[INFO] [1733933350.440251]: received mask_obs with header seq: 18848 +stamp: + secs: 1733933350 + nsecs: 440990209 +frame_id: '' and grasped_object_mask header seq: 450980 +stamp: + secs: 1733933350 + nsecs: 193476200 +frame_id: "camera_color_optical_frame" +[INFO] [1733933350.640266]: received mask_obs with header seq: 18849 +stamp: + secs: 1733933350 + nsecs: 641072988 +frame_id: '' and grasped_object_mask header seq: 450987 +stamp: + secs: 1733933350 + nsecs: 426614523 +frame_id: "camera_color_optical_frame" +[INFO] [1733933350.866961]: received mask_obs with header seq: 18850 +stamp: + secs: 1733933350 + nsecs: 868086576 +frame_id: '' and grasped_object_mask header seq: 450993 +stamp: + secs: 1733933350 + nsecs: 626446247 +frame_id: "camera_color_optical_frame" +[INFO] [1733933351.049672]: received mask_obs with header seq: 18851 +stamp: + secs: 1733933351 + nsecs: 50514459 +frame_id: '' and grasped_object_mask header seq: 450999 +stamp: + secs: 1733933350 + nsecs: 827203512 +frame_id: "camera_color_optical_frame" +[INFO] [1733933351.227911]: received mask_obs with header seq: 18852 +stamp: + secs: 1733933351 + nsecs: 228705883 +frame_id: '' and grasped_object_mask header seq: 451004 +stamp: + secs: 1733933350 + nsecs: 995672464 +frame_id: "camera_color_optical_frame" +[INFO] [1733933351.421612]: received mask_obs with header seq: 18853 +stamp: + secs: 1733933351 + nsecs: 422624111 +frame_id: '' and grasped_object_mask header seq: 451010 +stamp: + secs: 1733933351 + nsecs: 197913885 +frame_id: "camera_color_optical_frame" +[INFO] [1733933351.621332]: received mask_obs with header seq: 18854 +stamp: + secs: 1733933351 + nsecs: 622477293 +frame_id: '' and grasped_object_mask header seq: 451016 +stamp: + secs: 1733933351 + nsecs: 400241852 +frame_id: "camera_color_optical_frame" +[INFO] [1733933351.821167]: received mask_obs with header seq: 18855 +stamp: + secs: 1733933351 + nsecs: 822294235 +frame_id: '' and grasped_object_mask header seq: 451022 +stamp: + secs: 1733933351 + nsecs: 602657080 +frame_id: "camera_color_optical_frame" +[INFO] [1733933352.020989]: received mask_obs with header seq: 18856 +stamp: + secs: 1733933352 + nsecs: 22149801 +frame_id: '' and grasped_object_mask header seq: 451028 +stamp: + secs: 1733933351 + nsecs: 804470778 +frame_id: "camera_color_optical_frame" +[INFO] [1733933352.222778]: received mask_obs with header seq: 18857 +stamp: + secs: 1733933352 + nsecs: 223794937 +frame_id: '' and grasped_object_mask header seq: 451034 +stamp: + secs: 1733933352 + nsecs: 4088879 +frame_id: "camera_color_optical_frame" +[INFO] [1733933352.420746]: received mask_obs with header seq: 18858 +stamp: + secs: 1733933352 + nsecs: 421864986 +frame_id: '' and grasped_object_mask header seq: 451040 +stamp: + secs: 1733933352 + nsecs: 203696251 +frame_id: "camera_color_optical_frame" +[INFO] [1733933352.620650]: received mask_obs with header seq: 18859 +stamp: + secs: 1733933352 + nsecs: 621751070 +frame_id: '' and grasped_object_mask header seq: 451046 +stamp: + secs: 1733933352 + nsecs: 403291464 +frame_id: "camera_color_optical_frame" +[INFO] [1733933352.833789]: received mask_obs with header seq: 18860 +stamp: + secs: 1733933352 + nsecs: 834792852 +frame_id: '' and grasped_object_mask header seq: 451052 +stamp: + secs: 1733933352 + nsecs: 602874994 +frame_id: "camera_color_optical_frame" +[INFO] [1733933353.032793]: received mask_obs with header seq: 18861 +stamp: + secs: 1733933353 + nsecs: 33919334 +frame_id: '' and grasped_object_mask header seq: 451058 +stamp: + secs: 1733933352 + nsecs: 802458286 +frame_id: "camera_color_optical_frame" +[INFO] [1733933353.225960]: received mask_obs with header seq: 18862 +stamp: + secs: 1733933353 + nsecs: 226752996 +frame_id: '' and grasped_object_mask header seq: 451064 +stamp: + secs: 1733933353 + nsecs: 2066135 +frame_id: "camera_color_optical_frame" +[INFO] [1733933353.419916]: received mask_obs with header seq: 18863 +stamp: + secs: 1733933353 + nsecs: 421039342 +frame_id: '' and grasped_object_mask header seq: 451070 +stamp: + secs: 1733933353 + nsecs: 201662064 +frame_id: "camera_color_optical_frame" +[INFO] [1733933353.620137]: received mask_obs with header seq: 18864 +stamp: + secs: 1733933353 + nsecs: 620991706 +frame_id: '' and grasped_object_mask header seq: 451076 +stamp: + secs: 1733933353 + nsecs: 401247263 +frame_id: "camera_color_optical_frame" +[INFO] [1733933353.821859]: received mask_obs with header seq: 18865 +stamp: + secs: 1733933353 + nsecs: 822993516 +frame_id: '' and grasped_object_mask header seq: 451082 +stamp: + secs: 1733933353 + nsecs: 600819826 +frame_id: "camera_color_optical_frame" +[INFO] [1733933354.020569]: received mask_obs with header seq: 18866 +stamp: + secs: 1733933354 + nsecs: 21545410 +frame_id: '' and grasped_object_mask header seq: 451088 +stamp: + secs: 1733933353 + nsecs: 800394297 +frame_id: "camera_color_optical_frame" +[INFO] [1733933354.222070]: received mask_obs with header seq: 18867 +stamp: + secs: 1733933354 + nsecs: 223183870 +frame_id: '' and grasped_object_mask header seq: 451094 +stamp: + secs: 1733933354 + nsecs: 3576 +frame_id: "camera_color_optical_frame" +[INFO] [1733933354.421790]: received mask_obs with header seq: 18868 +stamp: + secs: 1733933354 + nsecs: 422759771 +frame_id: '' and grasped_object_mask header seq: 451100 +stamp: + secs: 1733933354 + nsecs: 199601412 +frame_id: "camera_color_optical_frame" +[INFO] [1733933354.462135]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1733933355.202316]: Stored episode 1. +[INFO] [1733933355.202533]: Saving video... +[INFO] [1733933355.417103]: Video saved! +Starting episode 1 +[INFO] [1733933355.427812]: resetting environment +[INFO] [1733933355.450486]: cleared current plan +[INFO] [1733933355.450697]: moving to home +[INFO] [1733933358.353632]: reached home +[INFO] [1733933358.354078]: reset action history +[INFO] [1733933361.112446]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1733933378.737421]: resetting environment +[INFO] [1733933378.739973]: cleared current plan +[INFO] [1733933378.740163]: moving to home +[INFO] [1733933379.741573]: reached home +[INFO] [1733933379.742009]: reset action history +[INFO] [1733933382.452273]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/episode_rosbags/episode_1_2024-12-11-11-09-42.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-736760.log +started roslaunch server http://158.130.50.37:34037/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [737143] +process[print_text-5]: started with pid [737144] +process[rosbag_record-6]: started with pid [737145] +started bagging! +[INFO] [1733933383.128534]: received mask_obs with header seq: 18869 +stamp: + secs: 1733933383 + nsecs: 129228353 +frame_id: '' and grasped_object_mask header seq: 451963 +stamp: + secs: 1733933382 + nsecs: 933615208 +frame_id: "camera_color_optical_frame" +[INFO] [1733933383.336616]: received mask_obs with header seq: 18870 +stamp: + secs: 1733933383 + nsecs: 337714910 +frame_id: '' and grasped_object_mask header seq: 451969 +stamp: + secs: 1733933383 + nsecs: 133446455 +frame_id: "camera_color_optical_frame" +[INFO] [1733933383.534085]: received mask_obs with header seq: 18871 +stamp: + secs: 1733933383 + nsecs: 534580230 +frame_id: '' and grasped_object_mask header seq: 451975 +stamp: + secs: 1733933383 + nsecs: 333277702 +frame_id: "camera_color_optical_frame" +[INFO] [1733933383.737611]: received mask_obs with header seq: 18872 +stamp: + secs: 1733933383 + nsecs: 738842010 +frame_id: '' and grasped_object_mask header seq: 451981 +stamp: + secs: 1733933383 + nsecs: 533108473 +frame_id: "camera_color_optical_frame" +[INFO] [1733933383.938810]: received mask_obs with header seq: 18873 +stamp: + secs: 1733933383 + nsecs: 939765453 +frame_id: '' and grasped_object_mask header seq: 451987 +stamp: + secs: 1733933383 + nsecs: 732939959 +frame_id: "camera_color_optical_frame" +[INFO] [1733933384.145233]: received mask_obs with header seq: 18874 +stamp: + secs: 1733933384 + nsecs: 146458864 +frame_id: '' and grasped_object_mask header seq: 451993 +stamp: + secs: 1733933383 + nsecs: 932779789 +frame_id: "camera_color_optical_frame" +[INFO] [1733933384.348073]: received mask_obs with header seq: 18875 +stamp: + secs: 1733933384 + nsecs: 347307205 +frame_id: '' and grasped_object_mask header seq: 451999 +stamp: + secs: 1733933384 + nsecs: 132621527 +frame_id: "camera_color_optical_frame" +[INFO] [1733933384.551017]: received mask_obs with header seq: 18876 +stamp: + secs: 1733933384 + nsecs: 552182197 +frame_id: '' and grasped_object_mask header seq: 452005 +stamp: + secs: 1733933384 + nsecs: 332463741 +frame_id: "camera_color_optical_frame" +[INFO] [1733933384.753333]: received mask_obs with header seq: 18877 +stamp: + secs: 1733933384 + nsecs: 754602193 +frame_id: '' and grasped_object_mask header seq: 452011 +stamp: + secs: 1733933384 + nsecs: 532305717 +frame_id: "camera_color_optical_frame" +[INFO] [1733933384.950340]: received mask_obs with header seq: 18878 +stamp: + secs: 1733933384 + nsecs: 951591253 +frame_id: '' and grasped_object_mask header seq: 452017 +stamp: + secs: 1733933384 + nsecs: 732147694 +frame_id: "camera_color_optical_frame" +[INFO] [1733933385.161101]: received mask_obs with header seq: 18879 +stamp: + secs: 1733933385 + nsecs: 156205654 +frame_id: '' and grasped_object_mask header seq: 452023 +stamp: + secs: 1733933384 + nsecs: 931986332 +frame_id: "camera_color_optical_frame" +[INFO] [1733933385.360156]: received mask_obs with header seq: 18880 +stamp: + secs: 1733933385 + nsecs: 347530603 +frame_id: '' and grasped_object_mask header seq: 452029 +stamp: + secs: 1733933385 + nsecs: 131823063 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -109069255 for nearest: 1733933385222590300 - target: 1733933385331659555 at idx 44 +[INFO] [1733933385.547469]: received mask_obs with header seq: 18881 +stamp: + secs: 1733933385 + nsecs: 548661708 +frame_id: '' and grasped_object_mask header seq: 452035 +stamp: + secs: 1733933385 + nsecs: 331659555 +frame_id: "camera_color_optical_frame" +[INFO] [1733933385.750557]: received mask_obs with header seq: 18882 +stamp: + secs: 1733933385 + nsecs: 751610040 +frame_id: '' and grasped_object_mask header seq: 452041 +stamp: + secs: 1733933385 + nsecs: 531496286 +frame_id: "camera_color_optical_frame" +[INFO] [1733933385.947864]: received mask_obs with header seq: 18883 +stamp: + secs: 1733933385 + nsecs: 949104070 +frame_id: '' and grasped_object_mask header seq: 452047 +stamp: + secs: 1733933385 + nsecs: 731333017 +frame_id: "camera_color_optical_frame" +[INFO] [1733933386.147875]: received mask_obs with header seq: 18884 +stamp: + secs: 1733933386 + nsecs: 148401260 +frame_id: '' and grasped_object_mask header seq: 452053 +stamp: + secs: 1733933385 + nsecs: 931169271 +frame_id: "camera_color_optical_frame" +[INFO] [1733933386.347508]: received mask_obs with header seq: 18885 +stamp: + secs: 1733933386 + nsecs: 348744392 +frame_id: '' and grasped_object_mask header seq: 452059 +stamp: + secs: 1733933386 + nsecs: 131005764 +frame_id: "camera_color_optical_frame" +[INFO] [1733933386.547544]: received mask_obs with header seq: 18886 +stamp: + secs: 1733933386 + nsecs: 547721385 +frame_id: '' and grasped_object_mask header seq: 452065 +stamp: + secs: 1733933386 + nsecs: 330841541 +frame_id: "camera_color_optical_frame" +[INFO] [1733933386.751842]: received mask_obs with header seq: 18887 +stamp: + secs: 1733933386 + nsecs: 752976179 +frame_id: '' and grasped_object_mask header seq: 452071 +stamp: + secs: 1733933386 + nsecs: 530677319 +frame_id: "camera_color_optical_frame" +[INFO] [1733933386.946613]: received mask_obs with header seq: 18888 +stamp: + secs: 1733933386 + nsecs: 947085857 +frame_id: '' and grasped_object_mask header seq: 452077 +stamp: + secs: 1733933386 + nsecs: 730512857 +frame_id: "camera_color_optical_frame" +[INFO] [1733933387.146446]: received mask_obs with header seq: 18889 +stamp: + secs: 1733933387 + nsecs: 147705078 +frame_id: '' and grasped_object_mask header seq: 452083 +stamp: + secs: 1733933386 + nsecs: 930348158 +frame_id: "camera_color_optical_frame" +[INFO] [1733933387.346280]: received mask_obs with header seq: 18890 +stamp: + secs: 1733933387 + nsecs: 346812248 +frame_id: '' and grasped_object_mask header seq: 452089 +stamp: + secs: 1733933387 + nsecs: 130183458 +frame_id: "camera_color_optical_frame" +[INFO] [1733933387.547082]: received mask_obs with header seq: 18891 +stamp: + secs: 1733933387 + nsecs: 546855449 +frame_id: '' and grasped_object_mask header seq: 452095 +stamp: + secs: 1733933387 + nsecs: 330019474 +frame_id: "camera_color_optical_frame" +[INFO] [1733933387.750606]: received mask_obs with header seq: 18892 +stamp: + secs: 1733933387 + nsecs: 751952886 +frame_id: '' and grasped_object_mask header seq: 452101 +stamp: + secs: 1733933387 + nsecs: 529854774 +frame_id: "camera_color_optical_frame" +[INFO] [1733933387.945971]: received mask_obs with header seq: 18893 +stamp: + secs: 1733933387 + nsecs: 946233272 +frame_id: '' and grasped_object_mask header seq: 452107 +stamp: + secs: 1733933387 + nsecs: 729689598 +frame_id: "camera_color_optical_frame" +[INFO] [1733933388.146308]: received mask_obs with header seq: 18894 +stamp: + secs: 1733933388 + nsecs: 147250890 +frame_id: '' and grasped_object_mask header seq: 452113 +stamp: + secs: 1733933387 + nsecs: 929525137 +frame_id: "camera_color_optical_frame" +[INFO] [1733933388.350521]: received mask_obs with header seq: 18895 +stamp: + secs: 1733933388 + nsecs: 351735115 +frame_id: '' and grasped_object_mask header seq: 452119 +stamp: + secs: 1733933388 + nsecs: 129360676 +frame_id: "camera_color_optical_frame" +[INFO] [1733933388.545311]: received mask_obs with header seq: 18896 +stamp: + secs: 1733933388 + nsecs: 546016693 +frame_id: '' and grasped_object_mask header seq: 452125 +stamp: + secs: 1733933388 + nsecs: 329195976 +frame_id: "camera_color_optical_frame" +[INFO] [1733933388.749207]: received mask_obs with header seq: 18897 +stamp: + secs: 1733933388 + nsecs: 750479698 +frame_id: '' and grasped_object_mask header seq: 452131 +stamp: + secs: 1733933388 + nsecs: 529032469 +frame_id: "camera_color_optical_frame" +[INFO] [1733933388.946010]: received mask_obs with header seq: 18898 +stamp: + secs: 1733933388 + nsecs: 945654630 +frame_id: '' and grasped_object_mask header seq: 452137 +stamp: + secs: 1733933388 + nsecs: 728867769 +frame_id: "camera_color_optical_frame" +[INFO] [1733933389.145990]: received mask_obs with header seq: 18899 +stamp: + secs: 1733933389 + nsecs: 146475791 +frame_id: '' and grasped_object_mask header seq: 452143 +stamp: + secs: 1733933388 + nsecs: 928709507 +frame_id: "camera_color_optical_frame" +[INFO] [1733933389.349123]: received mask_obs with header seq: 18900 +stamp: + secs: 1733933389 + nsecs: 349803924 +frame_id: '' and grasped_object_mask header seq: 452149 +stamp: + secs: 1733933389 + nsecs: 128552675 +frame_id: "camera_color_optical_frame" +[INFO] [1733933389.545305]: received mask_obs with header seq: 18901 +stamp: + secs: 1733933389 + nsecs: 546063661 +frame_id: '' and grasped_object_mask header seq: 452155 +stamp: + secs: 1733933389 + nsecs: 328396559 +frame_id: "camera_color_optical_frame" +[INFO] [1733933389.751275]: received mask_obs with header seq: 18902 +stamp: + secs: 1733933389 + nsecs: 752442598 +frame_id: '' and grasped_object_mask header seq: 452161 +stamp: + secs: 1733933389 + nsecs: 528240442 +frame_id: "camera_color_optical_frame" +[INFO] [1733933389.945909]: received mask_obs with header seq: 18903 +stamp: + secs: 1733933389 + nsecs: 946796178 +frame_id: '' and grasped_object_mask header seq: 452167 +stamp: + secs: 1733933389 + nsecs: 728085995 +frame_id: "camera_color_optical_frame" +[INFO] [1733933390.145260]: received mask_obs with header seq: 18904 +stamp: + secs: 1733933390 + nsecs: 145911455 +frame_id: '' and grasped_object_mask header seq: 452173 +stamp: + secs: 1733933389 + nsecs: 927931786 +frame_id: "camera_color_optical_frame" +[INFO] [1733933390.348155]: received mask_obs with header seq: 18905 +stamp: + secs: 1733933390 + nsecs: 349456548 +frame_id: '' and grasped_object_mask header seq: 452179 +stamp: + secs: 1733933390 + nsecs: 127778530 +frame_id: "camera_color_optical_frame" +[INFO] [1733933390.545302]: received mask_obs with header seq: 18906 +stamp: + secs: 1733933390 + nsecs: 546471118 +frame_id: '' and grasped_object_mask header seq: 452185 +stamp: + secs: 1733933390 + nsecs: 327625513 +frame_id: "camera_color_optical_frame" +[INFO] [1733933390.751344]: received mask_obs with header seq: 18907 +stamp: + secs: 1733933390 + nsecs: 752393484 +frame_id: '' and grasped_object_mask header seq: 452191 +stamp: + secs: 1733933390 + nsecs: 527472973 +frame_id: "camera_color_optical_frame" +[INFO] [1733933390.951164]: received mask_obs with header seq: 18908 +stamp: + secs: 1733933390 + nsecs: 944504022 +frame_id: '' and grasped_object_mask header seq: 452197 +stamp: + secs: 1733933390 + nsecs: 727320671 +frame_id: "camera_color_optical_frame" +[INFO] [1733933391.145843]: received mask_obs with header seq: 18909 +stamp: + secs: 1733933391 + nsecs: 145615577 +frame_id: '' and grasped_object_mask header seq: 452203 +stamp: + secs: 1733933390 + nsecs: 927163124 +frame_id: "camera_color_optical_frame" +[INFO] [1733933391.345646]: received mask_obs with header seq: 18910 +stamp: + secs: 1733933391 + nsecs: 345785379 +frame_id: '' and grasped_object_mask header seq: 452209 +stamp: + secs: 1733933391 + nsecs: 127005339 +frame_id: "camera_color_optical_frame" +[INFO] [1733933391.545218]: received mask_obs with header seq: 18911 +stamp: + secs: 1733933391 + nsecs: 546288728 +frame_id: '' and grasped_object_mask header seq: 452215 +stamp: + secs: 1733933391 + nsecs: 326846600 +frame_id: "camera_color_optical_frame" +[INFO] [1733933391.749807]: received mask_obs with header seq: 18912 +stamp: + secs: 1733933391 + nsecs: 750948905 +frame_id: '' and grasped_object_mask header seq: 452221 +stamp: + secs: 1733933391 + nsecs: 526688337 +frame_id: "camera_color_optical_frame" +[INFO] [1733933391.954613]: received mask_obs with header seq: 18913 +stamp: + secs: 1733933391 + nsecs: 945201158 +frame_id: '' and grasped_object_mask header seq: 452227 +stamp: + secs: 1733933391 + nsecs: 726529837 +frame_id: "camera_color_optical_frame" +[INFO] [1733933392.153825]: received mask_obs with header seq: 18914 +stamp: + secs: 1733933392 + nsecs: 154072523 +frame_id: '' and grasped_object_mask header seq: 452233 +stamp: + secs: 1733933391 + nsecs: 926368475 +frame_id: "camera_color_optical_frame" +[INFO] [1733933392.349935]: received mask_obs with header seq: 18915 +stamp: + secs: 1733933392 + nsecs: 350918292 +frame_id: '' and grasped_object_mask header seq: 452239 +stamp: + secs: 1733933392 + nsecs: 126206636 +frame_id: "camera_color_optical_frame" +[INFO] [1733933392.549431]: received mask_obs with header seq: 18916 +stamp: + secs: 1733933392 + nsecs: 550756692 +frame_id: '' and grasped_object_mask header seq: 452245 +stamp: + secs: 1733933392 + nsecs: 326045752 +frame_id: "camera_color_optical_frame" +[INFO] [1733933392.750190]: received mask_obs with header seq: 18917 +stamp: + secs: 1733933392 + nsecs: 750325918 +frame_id: '' and grasped_object_mask header seq: 452251 +stamp: + secs: 1733933392 + nsecs: 525883675 +frame_id: "camera_color_optical_frame" +[INFO] [1733933392.945020]: received mask_obs with header seq: 18918 +stamp: + secs: 1733933392 + nsecs: 945891141 +frame_id: '' and grasped_object_mask header seq: 452257 +stamp: + secs: 1733933392 + nsecs: 725721836 +frame_id: "camera_color_optical_frame" +[INFO] [1733933393.145064]: received mask_obs with header seq: 18919 +stamp: + secs: 1733933393 + nsecs: 146355867 +frame_id: '' and grasped_object_mask header seq: 452263 +stamp: + secs: 1733933392 + nsecs: 925561428 +frame_id: "camera_color_optical_frame" +[INFO] [1733933393.354106]: received mask_obs with header seq: 18920 +stamp: + secs: 1733933393 + nsecs: 345217943 +frame_id: '' and grasped_object_mask header seq: 452269 +stamp: + secs: 1733933393 + nsecs: 125401497 +frame_id: "camera_color_optical_frame" +[INFO] [1733933393.567326]: received mask_obs with header seq: 18921 +stamp: + secs: 1733933393 + nsecs: 568568944 +frame_id: '' and grasped_object_mask header seq: 452275 +stamp: + secs: 1733933393 + nsecs: 325241804 +frame_id: "camera_color_optical_frame" +[INFO] [1733933393.762253]: received mask_obs with header seq: 18922 +stamp: + secs: 1733933393 + nsecs: 762710332 +frame_id: '' and grasped_object_mask header seq: 452281 +stamp: + secs: 1733933393 + nsecs: 525082827 +frame_id: "camera_color_optical_frame" +[INFO] [1733933393.958075]: received mask_obs with header seq: 18923 +stamp: + secs: 1733933393 + nsecs: 959341764 +frame_id: '' and grasped_object_mask header seq: 452287 +stamp: + secs: 1733933393 + nsecs: 724922895 +frame_id: "camera_color_optical_frame" +[INFO] [1733933394.152702]: received mask_obs with header seq: 18924 +stamp: + secs: 1733933394 + nsecs: 154027938 +frame_id: '' and grasped_object_mask header seq: 452293 +stamp: + secs: 1733933393 + nsecs: 924762726 +frame_id: "camera_color_optical_frame" +[INFO] [1733933394.344389]: received mask_obs with header seq: 18925 +stamp: + secs: 1733933394 + nsecs: 344853162 +frame_id: '' and grasped_object_mask header seq: 452299 +stamp: + secs: 1733933394 + nsecs: 124602795 +frame_id: "camera_color_optical_frame" +[INFO] [1733933394.558777]: received mask_obs with header seq: 18926 +stamp: + secs: 1733933394 + nsecs: 559828996 +frame_id: '' and grasped_object_mask header seq: 452305 +stamp: + secs: 1733933394 + nsecs: 324442625 +frame_id: "camera_color_optical_frame" +[INFO] [1733933394.752699]: received mask_obs with header seq: 18927 +stamp: + secs: 1733933394 + nsecs: 753338575 +frame_id: '' and grasped_object_mask header seq: 452311 +stamp: + secs: 1733933394 + nsecs: 524282694 +frame_id: "camera_color_optical_frame" +[INFO] [1733933394.958125]: received mask_obs with header seq: 18928 +stamp: + secs: 1733933394 + nsecs: 948989629 +frame_id: '' and grasped_object_mask header seq: 452317 +stamp: + secs: 1733933394 + nsecs: 724123478 +frame_id: "camera_color_optical_frame" +[INFO] [1733933395.159033]: received mask_obs with header seq: 18929 +stamp: + secs: 1733933395 + nsecs: 159985065 +frame_id: '' and grasped_object_mask header seq: 452323 +stamp: + secs: 1733933394 + nsecs: 923958302 +frame_id: "camera_color_optical_frame" +[INFO] [1733933395.354922]: received mask_obs with header seq: 18930 +stamp: + secs: 1733933395 + nsecs: 356191396 +frame_id: '' and grasped_object_mask header seq: 452329 +stamp: + secs: 1733933395 + nsecs: 123790979 +frame_id: "camera_color_optical_frame" +[INFO] [1733933395.548938]: received mask_obs with header seq: 18931 +stamp: + secs: 1733933395 + nsecs: 549890995 +frame_id: '' and grasped_object_mask header seq: 452335 +stamp: + secs: 1733933395 + nsecs: 323623896 +frame_id: "camera_color_optical_frame" +[INFO] [1733933395.747667]: received mask_obs with header seq: 18932 +stamp: + secs: 1733933395 + nsecs: 749012708 +frame_id: '' and grasped_object_mask header seq: 452341 +stamp: + secs: 1733933395 + nsecs: 523456335 +frame_id: "camera_color_optical_frame" +[INFO] [1733933395.944626]: received mask_obs with header seq: 18933 +stamp: + secs: 1733933395 + nsecs: 944958448 +frame_id: '' and grasped_object_mask header seq: 452347 +stamp: + secs: 1733933395 + nsecs: 723288536 +frame_id: "camera_color_optical_frame" +[INFO] [1733933396.149466]: received mask_obs with header seq: 18934 +stamp: + secs: 1733933396 + nsecs: 150342702 +frame_id: '' and grasped_object_mask header seq: 452353 +stamp: + secs: 1733933395 + nsecs: 923129082 +frame_id: "camera_color_optical_frame" +[INFO] [1733933396.343793]: received mask_obs with header seq: 18935 +stamp: + secs: 1733933396 + nsecs: 344662189 +frame_id: '' and grasped_object_mask header seq: 452359 +stamp: + secs: 1733933396 + nsecs: 122971058 +frame_id: "camera_color_optical_frame" +[INFO] [1733933396.548053]: received mask_obs with header seq: 18936 +stamp: + secs: 1733933396 + nsecs: 548702716 +frame_id: '' and grasped_object_mask header seq: 452365 +stamp: + secs: 1733933396 + nsecs: 322812796 +frame_id: "camera_color_optical_frame" +[INFO] [1733933396.748409]: received mask_obs with header seq: 18937 +stamp: + secs: 1733933396 + nsecs: 748881578 +frame_id: '' and grasped_object_mask header seq: 452371 +stamp: + secs: 1733933396 + nsecs: 522654772 +frame_id: "camera_color_optical_frame" +[INFO] [1733933396.943364]: received mask_obs with header seq: 18938 +stamp: + secs: 1733933396 + nsecs: 944450139 +frame_id: '' and grasped_object_mask header seq: 452377 +stamp: + secs: 1733933396 + nsecs: 722496986 +frame_id: "camera_color_optical_frame" +[INFO] [1733933397.156561]: received mask_obs with header seq: 18939 +stamp: + secs: 1733933397 + nsecs: 157852888 +frame_id: '' and grasped_object_mask header seq: 452383 +stamp: + secs: 1733933396 + nsecs: 922334433 +frame_id: "camera_color_optical_frame" +[INFO] [1733933397.343620]: received mask_obs with header seq: 18940 +stamp: + secs: 1733933397 + nsecs: 344968795 +frame_id: '' and grasped_object_mask header seq: 452389 +stamp: + secs: 1733933397 + nsecs: 122171640 +frame_id: "camera_color_optical_frame" +[INFO] [1733933397.547678]: received mask_obs with header seq: 18941 +stamp: + secs: 1733933397 + nsecs: 548138856 +frame_id: '' and grasped_object_mask header seq: 452395 +stamp: + secs: 1733933397 + nsecs: 322007895 +frame_id: "camera_color_optical_frame" +[INFO] [1733933397.749648]: received mask_obs with header seq: 18942 +stamp: + secs: 1733933397 + nsecs: 750153779 +frame_id: '' and grasped_object_mask header seq: 452401 +stamp: + secs: 1733933397 + nsecs: 521843672 +frame_id: "camera_color_optical_frame" +[INFO] [1733933397.955616]: received mask_obs with header seq: 18943 +stamp: + secs: 1733933397 + nsecs: 956944704 +frame_id: '' and grasped_object_mask header seq: 452407 +stamp: + secs: 1733933397 + nsecs: 721679926 +frame_id: "camera_color_optical_frame" +[INFO] [1733933398.146576]: received mask_obs with header seq: 18944 +stamp: + secs: 1733933398 + nsecs: 147006750 +frame_id: '' and grasped_object_mask header seq: 452413 +stamp: + secs: 1733933397 + nsecs: 921521902 +frame_id: "camera_color_optical_frame" +[INFO] [1733933398.348566]: received mask_obs with header seq: 18945 +stamp: + secs: 1733933398 + nsecs: 348889112 +frame_id: '' and grasped_object_mask header seq: 452419 +stamp: + secs: 1733933398 + nsecs: 121365786 +frame_id: "camera_color_optical_frame" +[INFO] [1733933398.548896]: received mask_obs with header seq: 18946 +stamp: + secs: 1733933398 + nsecs: 548941373 +frame_id: '' and grasped_object_mask header seq: 452425 +stamp: + secs: 1733933398 + nsecs: 321209908 +frame_id: "camera_color_optical_frame" +[INFO] [1733933398.750113]: received mask_obs with header seq: 18947 +stamp: + secs: 1733933398 + nsecs: 750493526 +frame_id: '' and grasped_object_mask header seq: 452431 +stamp: + secs: 1733933398 + nsecs: 521055460 +frame_id: "camera_color_optical_frame" +[INFO] [1733933398.947524]: received mask_obs with header seq: 18948 +stamp: + secs: 1733933398 + nsecs: 948672056 +frame_id: '' and grasped_object_mask header seq: 452437 +stamp: + secs: 1733933398 + nsecs: 720900059 +frame_id: "camera_color_optical_frame" +[INFO] [1733933399.148657]: received mask_obs with header seq: 18949 +stamp: + secs: 1733933399 + nsecs: 149954080 +frame_id: '' and grasped_object_mask header seq: 452443 +stamp: + secs: 1733933398 + nsecs: 920764685 +frame_id: "camera_color_optical_frame" +[INFO] [1733933399.349463]: received mask_obs with header seq: 18950 +stamp: + secs: 1733933399 + nsecs: 350755929 +frame_id: '' and grasped_object_mask header seq: 452449 +stamp: + secs: 1733933399 + nsecs: 120635509 +frame_id: "camera_color_optical_frame" +[INFO] [1733933399.555643]: received mask_obs with header seq: 18951 +stamp: + secs: 1733933399 + nsecs: 556954622 +frame_id: '' and grasped_object_mask header seq: 452455 +stamp: + secs: 1733933399 + nsecs: 320507526 +frame_id: "camera_color_optical_frame" +[INFO] [1733933399.749896]: received mask_obs with header seq: 18952 +stamp: + secs: 1733933399 + nsecs: 751160383 +frame_id: '' and grasped_object_mask header seq: 452461 +stamp: + secs: 1733933399 + nsecs: 520380974 +frame_id: "camera_color_optical_frame" +[INFO] [1733933399.946955]: received mask_obs with header seq: 18953 +stamp: + secs: 1733933399 + nsecs: 948107481 +frame_id: '' and grasped_object_mask header seq: 452467 +stamp: + secs: 1733933399 + nsecs: 720256090 +frame_id: "camera_color_optical_frame" +[INFO] [1733933400.143702]: received mask_obs with header seq: 18954 +stamp: + secs: 1733933400 + nsecs: 144579648 +frame_id: '' and grasped_object_mask header seq: 452473 +stamp: + secs: 1733933399 + nsecs: 920099735 +frame_id: "camera_color_optical_frame" +[INFO] [1733933400.356949]: received mask_obs with header seq: 18955 +stamp: + secs: 1733933400 + nsecs: 358250379 +frame_id: '' and grasped_object_mask header seq: 452479 +stamp: + secs: 1733933400 + nsecs: 119934797 +frame_id: "camera_color_optical_frame" +[INFO] [1733933400.550902]: received mask_obs with header seq: 18956 +stamp: + secs: 1733933400 + nsecs: 552215576 +frame_id: '' and grasped_object_mask header seq: 452485 +stamp: + secs: 1733933400 + nsecs: 319769621 +frame_id: "camera_color_optical_frame" +[INFO] [1733933400.748529]: received mask_obs with header seq: 18957 +stamp: + secs: 1733933400 + nsecs: 748784303 +frame_id: '' and grasped_object_mask header seq: 452491 +stamp: + secs: 1733933400 + nsecs: 519604206 +frame_id: "camera_color_optical_frame" +[INFO] [1733933400.948266]: received mask_obs with header seq: 18958 +stamp: + secs: 1733933400 + nsecs: 948598861 +frame_id: '' and grasped_object_mask header seq: 452497 +stamp: + secs: 1733933400 + nsecs: 719438791 +frame_id: "camera_color_optical_frame" +[INFO] [1733933401.146710]: received mask_obs with header seq: 18959 +stamp: + secs: 1733933401 + nsecs: 148106098 +frame_id: '' and grasped_object_mask header seq: 452503 +stamp: + secs: 1733933400 + nsecs: 919271469 +frame_id: "camera_color_optical_frame" +[INFO] [1733933401.346454]: received mask_obs with header seq: 18960 +stamp: + secs: 1733933401 + nsecs: 347663402 +frame_id: '' and grasped_object_mask header seq: 452509 +stamp: + secs: 1733933401 + nsecs: 119101763 +frame_id: "camera_color_optical_frame" +[INFO] [1733933401.544988]: received mask_obs with header seq: 18961 +stamp: + secs: 1733933401 + nsecs: 545876741 +frame_id: '' and grasped_object_mask header seq: 452515 +stamp: + secs: 1733933401 + nsecs: 318931580 +frame_id: "camera_color_optical_frame" +[INFO] [1733933401.747756]: received mask_obs with header seq: 18962 +stamp: + secs: 1733933401 + nsecs: 749083042 +frame_id: '' and grasped_object_mask header seq: 452521 +stamp: + secs: 1733933401 + nsecs: 518761158 +frame_id: "camera_color_optical_frame" +[INFO] [1733933401.947919]: received mask_obs with header seq: 18963 +stamp: + secs: 1733933401 + nsecs: 948068380 +frame_id: '' and grasped_object_mask header seq: 452527 +stamp: + secs: 1733933401 + nsecs: 718590260 +frame_id: "camera_color_optical_frame" +[INFO] [1733933402.157770]: received mask_obs with header seq: 18964 +stamp: + secs: 1733933402 + nsecs: 159189224 +frame_id: '' and grasped_object_mask header seq: 452533 +stamp: + secs: 1733933401 + nsecs: 918418646 +frame_id: "camera_color_optical_frame" +[INFO] [1733933402.348652]: received mask_obs with header seq: 18965 +stamp: + secs: 1733933402 + nsecs: 348931074 +frame_id: '' and grasped_object_mask header seq: 452539 +stamp: + secs: 1733933402 + nsecs: 118247747 +frame_id: "camera_color_optical_frame" +[INFO] [1733933402.547964]: received mask_obs with header seq: 18966 +stamp: + secs: 1733933402 + nsecs: 548291444 +frame_id: '' and grasped_object_mask header seq: 452545 +stamp: + secs: 1733933402 + nsecs: 318075418 +frame_id: "camera_color_optical_frame" +[INFO] [1733933402.749126]: received mask_obs with header seq: 18967 +stamp: + secs: 1733933402 + nsecs: 749333858 +frame_id: '' and grasped_object_mask header seq: 452551 +stamp: + secs: 1733933402 + nsecs: 517902613 +frame_id: "camera_color_optical_frame" +[INFO] [1733933402.949163]: received mask_obs with header seq: 18968 +stamp: + secs: 1733933402 + nsecs: 949326992 +frame_id: '' and grasped_object_mask header seq: 452557 +stamp: + secs: 1733933402 + nsecs: 717729330 +frame_id: "camera_color_optical_frame" +[INFO] [1733933403.147530]: received mask_obs with header seq: 18969 +stamp: + secs: 1733933403 + nsecs: 147986412 +frame_id: '' and grasped_object_mask header seq: 452563 +stamp: + secs: 1733933402 + nsecs: 917559862 +frame_id: "camera_color_optical_frame" +[INFO] [1733933403.357065]: received mask_obs with header seq: 18970 +stamp: + secs: 1733933403 + nsecs: 358290910 +frame_id: '' and grasped_object_mask header seq: 452569 +stamp: + secs: 1733933403 + nsecs: 117391586 +frame_id: "camera_color_optical_frame" +[INFO] [1733933403.546110]: received mask_obs with header seq: 18971 +stamp: + secs: 1733933403 + nsecs: 547335386 +frame_id: '' and grasped_object_mask header seq: 452575 +stamp: + secs: 1733933403 + nsecs: 317223549 +frame_id: "camera_color_optical_frame" +[INFO] [1733933403.746482]: received mask_obs with header seq: 18972 +stamp: + secs: 1733933403 + nsecs: 747641086 +frame_id: '' and grasped_object_mask header seq: 452581 +stamp: + secs: 1733933403 + nsecs: 517054558 +frame_id: "camera_color_optical_frame" +[INFO] [1733933403.945886]: received mask_obs with header seq: 18973 +stamp: + secs: 1733933403 + nsecs: 947263240 +frame_id: '' and grasped_object_mask header seq: 452587 +stamp: + secs: 1733933403 + nsecs: 716885090 +frame_id: "camera_color_optical_frame" +[INFO] [1733933404.147337]: received mask_obs with header seq: 18974 +stamp: + secs: 1733933404 + nsecs: 147816658 +frame_id: '' and grasped_object_mask header seq: 452593 +stamp: + secs: 1733933403 + nsecs: 916717052 +frame_id: "camera_color_optical_frame" +[INFO] [1733933404.348270]: received mask_obs with header seq: 18975 +stamp: + secs: 1733933404 + nsecs: 349400043 +frame_id: '' and grasped_object_mask header seq: 452599 +stamp: + secs: 1733933404 + nsecs: 116549253 +frame_id: "camera_color_optical_frame" +[INFO] [1733933404.549227]: received mask_obs with header seq: 18976 +stamp: + secs: 1733933404 + nsecs: 550063610 +frame_id: '' and grasped_object_mask header seq: 452605 +stamp: + secs: 1733933404 + nsecs: 316381216 +frame_id: "camera_color_optical_frame" +[INFO] [1733933404.751283]: received mask_obs with header seq: 18977 +stamp: + secs: 1733933404 + nsecs: 752019643 +frame_id: '' and grasped_object_mask header seq: 452611 +stamp: + secs: 1733933404 + nsecs: 516213655 +frame_id: "camera_color_optical_frame" +[INFO] [1733933404.945921]: received mask_obs with header seq: 18978 +stamp: + secs: 1733933404 + nsecs: 947248935 +frame_id: '' and grasped_object_mask header seq: 452617 +stamp: + secs: 1733933404 + nsecs: 716044664 +frame_id: "camera_color_optical_frame" +[INFO] [1733933405.147792]: received mask_obs with header seq: 18979 +stamp: + secs: 1733933405 + nsecs: 149129867 +frame_id: '' and grasped_object_mask header seq: 452623 +stamp: + secs: 1733933404 + nsecs: 915882826 +frame_id: "camera_color_optical_frame" +[INFO] [1733933405.345864]: received mask_obs with header seq: 18980 +stamp: + secs: 1733933405 + nsecs: 347179889 +frame_id: '' and grasped_object_mask header seq: 452629 +stamp: + secs: 1733933405 + nsecs: 115722895 +frame_id: "camera_color_optical_frame" +[INFO] [1733933405.546405]: received mask_obs with header seq: 18981 +stamp: + secs: 1733933405 + nsecs: 547544002 +frame_id: '' and grasped_object_mask header seq: 452635 +stamp: + secs: 1733933405 + nsecs: 315563202 +frame_id: "camera_color_optical_frame" +[INFO] [1733933405.748249]: received mask_obs with header seq: 18982 +stamp: + secs: 1733933405 + nsecs: 749531984 +frame_id: '' and grasped_object_mask header seq: 452641 +stamp: + secs: 1733933405 + nsecs: 515403509 +frame_id: "camera_color_optical_frame" +[INFO] [1733933405.945982]: received mask_obs with header seq: 18983 +stamp: + secs: 1733933405 + nsecs: 947237730 +frame_id: '' and grasped_object_mask header seq: 452647 +stamp: + secs: 1733933405 + nsecs: 715243816 +frame_id: "camera_color_optical_frame" +[INFO] [1733933406.148897]: received mask_obs with header seq: 18984 +stamp: + secs: 1733933406 + nsecs: 150220870 +frame_id: '' and grasped_object_mask header seq: 452653 +stamp: + secs: 1733933405 + nsecs: 915088177 +frame_id: "camera_color_optical_frame" +[INFO] [1733933406.346925]: received mask_obs with header seq: 18985 +stamp: + secs: 1733933406 + nsecs: 348262310 +frame_id: '' and grasped_object_mask header seq: 452659 +stamp: + secs: 1733933406 + nsecs: 114932775 +frame_id: "camera_color_optical_frame" +[INFO] [1733933406.548884]: received mask_obs with header seq: 18986 +stamp: + secs: 1733933406 + nsecs: 550135850 +frame_id: '' and grasped_object_mask header seq: 452665 +stamp: + secs: 1733933406 + nsecs: 314777613 +frame_id: "camera_color_optical_frame" +[INFO] [1733933406.751775]: received mask_obs with header seq: 18987 +stamp: + secs: 1733933406 + nsecs: 753156185 +frame_id: '' and grasped_object_mask header seq: 452671 +stamp: + secs: 1733933406 + nsecs: 514623165 +frame_id: "camera_color_optical_frame" +[INFO] [1733933406.949061]: received mask_obs with header seq: 18988 +stamp: + secs: 1733933406 + nsecs: 950019359 +frame_id: '' and grasped_object_mask header seq: 452677 +stamp: + secs: 1733933406 + nsecs: 714468479 +frame_id: "camera_color_optical_frame" +[INFO] [1733933407.159651]: received mask_obs with header seq: 18989 +stamp: + secs: 1733933407 + nsecs: 160537242 +frame_id: '' and grasped_object_mask header seq: 452683 +stamp: + secs: 1733933406 + nsecs: 914310455 +frame_id: "camera_color_optical_frame" +[INFO] [1733933407.352669]: received mask_obs with header seq: 18990 +stamp: + secs: 1733933407 + nsecs: 354021787 +frame_id: '' and grasped_object_mask header seq: 452689 +stamp: + secs: 1733933407 + nsecs: 114152431 +frame_id: "camera_color_optical_frame" +[INFO] [1733933407.547144]: received mask_obs with header seq: 18991 +stamp: + secs: 1733933407 + nsecs: 548142910 +frame_id: '' and grasped_object_mask header seq: 452695 +stamp: + secs: 1733933407 + nsecs: 313993216 +frame_id: "camera_color_optical_frame" +[INFO] [1733933407.748544]: received mask_obs with header seq: 18992 +stamp: + secs: 1733933407 + nsecs: 749577522 +frame_id: '' and grasped_object_mask header seq: 452701 +stamp: + secs: 1733933407 + nsecs: 513834238 +frame_id: "camera_color_optical_frame" +[INFO] [1733933407.946905]: received mask_obs with header seq: 18993 +stamp: + secs: 1733933407 + nsecs: 948186159 +frame_id: '' and grasped_object_mask header seq: 452707 +stamp: + secs: 1733933407 + nsecs: 713675499 +frame_id: "camera_color_optical_frame" +[INFO] [1733933408.145808]: received mask_obs with header seq: 18994 +stamp: + secs: 1733933408 + nsecs: 147128582 +frame_id: '' and grasped_object_mask header seq: 452713 +stamp: + secs: 1733933407 + nsecs: 914477110 +frame_id: "camera_color_optical_frame" +[INFO] [1733933408.346609]: received mask_obs with header seq: 18995 +stamp: + secs: 1733933408 + nsecs: 347907066 +frame_id: '' and grasped_object_mask header seq: 452719 +stamp: + secs: 1733933408 + nsecs: 116664648 +frame_id: "camera_color_optical_frame" +[INFO] [1733933408.547669]: received mask_obs with header seq: 18996 +stamp: + secs: 1733933408 + nsecs: 549027442 +frame_id: '' and grasped_object_mask header seq: 452725 +stamp: + secs: 1733933408 + nsecs: 318940639 +frame_id: "camera_color_optical_frame" +[INFO] [1733933408.751553]: received mask_obs with header seq: 18997 +stamp: + secs: 1733933408 + nsecs: 752764940 +frame_id: '' and grasped_object_mask header seq: 452731 +stamp: + secs: 1733933408 + nsecs: 521303177 +frame_id: "camera_color_optical_frame" +[INFO] [1733933408.944863]: received mask_obs with header seq: 18998 +stamp: + secs: 1733933408 + nsecs: 946118354 +frame_id: '' and grasped_object_mask header seq: 452737 +stamp: + secs: 1733933408 + nsecs: 723753214 +frame_id: "camera_color_optical_frame" +[INFO] [1733933409.154565]: received mask_obs with header seq: 18999 +stamp: + secs: 1733933409 + nsecs: 155926942 +frame_id: '' and grasped_object_mask header seq: 452743 +stamp: + secs: 1733933408 + nsecs: 925063610 +frame_id: "camera_color_optical_frame" +[INFO] [1733933409.345480]: received mask_obs with header seq: 19000 +stamp: + secs: 1733933409 + nsecs: 346674680 +frame_id: '' and grasped_object_mask header seq: 452749 +stamp: + secs: 1733933409 + nsecs: 124684334 +frame_id: "camera_color_optical_frame" +[INFO] [1733933409.543654]: received mask_obs with header seq: 19001 +stamp: + secs: 1733933409 + nsecs: 544853687 +frame_id: '' and grasped_object_mask header seq: 452755 +stamp: + secs: 1733933409 + nsecs: 324293852 +frame_id: "camera_color_optical_frame" +[INFO] [1733933409.750899]: received mask_obs with header seq: 19002 +stamp: + secs: 1733933409 + nsecs: 752032518 +frame_id: '' and grasped_object_mask header seq: 452761 +stamp: + secs: 1733933409 + nsecs: 523893118 +frame_id: "camera_color_optical_frame" +[INFO] [1733933409.948745]: received mask_obs with header seq: 19003 +stamp: + secs: 1733933409 + nsecs: 949886560 +frame_id: '' and grasped_object_mask header seq: 452767 +stamp: + secs: 1733933409 + nsecs: 723479748 +frame_id: "camera_color_optical_frame" +[INFO] [1733933409.999813]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1733933410.154094]: received mask_obs with header seq: 19004 +stamp: + secs: 1733933410 + nsecs: 148189067 +frame_id: '' and grasped_object_mask header seq: 452773 +stamp: + secs: 1733933409 + nsecs: 923071623 +frame_id: "camera_color_optical_frame" +[INFO] [1733933410.793004]: Stored episode 2. +[INFO] [1733933410.793771]: Saving video... +[WARN] [1733933411.361340]: Plan exhausted +[WARN] [1733933411.402711]: Plan exhausted +[WARN] [1733933411.440367]: Plan exhausted +[INFO] [1733933411.394808]: Video saved! +Starting episode 2 +[INFO] [1733933411.429535]: resetting environment +[INFO] [1733933411.440964]: cleared current plan +[INFO] [1733933411.443573]: moving to home +[INFO] [1733933414.246178]: reached home +[INFO] [1733933414.251268]: reset action history +[INFO] [1733933417.026356]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1733933425.168974]: resetting environment +[INFO] [1733933425.187450]: cleared current plan +[INFO] [1733933425.192523]: moving to home +[INFO] [1733933426.197933]: reached home +[INFO] [1733933426.203267]: reset action history +[INFO] [1733933428.925596]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/episode_rosbags/episode_2_2024-12-11-11-10-28.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-736760.log +started roslaunch server http://158.130.50.37:39581/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [737305] +process[print_text-8]: started with pid [737306] +process[rosbag_record-9]: started with pid [737307] +started bagging! +[INFO] [1733933430.665927]: received mask_obs with header seq: 19005 +stamp: + secs: 1733933430 + nsecs: 667121887 +frame_id: '' and grasped_object_mask header seq: 453390 +stamp: + secs: 1733933430 + nsecs: 464898586 +frame_id: "camera_color_optical_frame" +[INFO] [1733933430.860556]: received mask_obs with header seq: 19006 +stamp: + secs: 1733933430 + nsecs: 861769676 +frame_id: '' and grasped_object_mask header seq: 453396 +stamp: + secs: 1733933430 + nsecs: 664484978 +frame_id: "camera_color_optical_frame" +[INFO] [1733933431.062315]: received mask_obs with header seq: 19007 +stamp: + secs: 1733933431 + nsecs: 63730478 +frame_id: '' and grasped_object_mask header seq: 453402 +stamp: + secs: 1733933430 + nsecs: 864059687 +frame_id: "camera_color_optical_frame" +[INFO] [1733933431.277906]: received mask_obs with header seq: 19008 +stamp: + secs: 1733933431 + nsecs: 279063701 +frame_id: '' and grasped_object_mask header seq: 453408 +stamp: + secs: 1733933431 + nsecs: 63668966 +frame_id: "camera_color_optical_frame" +[INFO] [1733933431.472985]: received mask_obs with header seq: 19009 +stamp: + secs: 1733933431 + nsecs: 474264144 +frame_id: '' and grasped_object_mask header seq: 453414 +stamp: + secs: 1733933431 + nsecs: 263273001 +frame_id: "camera_color_optical_frame" +[INFO] [1733933431.685833]: received mask_obs with header seq: 19010 +stamp: + secs: 1733933431 + nsecs: 686891317 +frame_id: '' and grasped_object_mask header seq: 453420 +stamp: + secs: 1733933431 + nsecs: 462865353 +frame_id: "camera_color_optical_frame" +[INFO] [1733933431.876978]: received mask_obs with header seq: 19011 +stamp: + secs: 1733933431 + nsecs: 878456830 +frame_id: '' and grasped_object_mask header seq: 453426 +stamp: + secs: 1733933431 + nsecs: 662445784 +frame_id: "camera_color_optical_frame" +[INFO] [1733933432.082322]: received mask_obs with header seq: 19012 +stamp: + secs: 1733933432 + nsecs: 83657741 +frame_id: '' and grasped_object_mask header seq: 453432 +stamp: + secs: 1733933431 + nsecs: 862014532 +frame_id: "camera_color_optical_frame" +[INFO] [1733933432.286286]: received mask_obs with header seq: 19013 +stamp: + secs: 1733933432 + nsecs: 287386655 +frame_id: '' and grasped_object_mask header seq: 453438 +stamp: + secs: 1733933432 + nsecs: 61627150 +frame_id: "camera_color_optical_frame" +[INFO] [1733933432.481914]: received mask_obs with header seq: 19014 +stamp: + secs: 1733933432 + nsecs: 482304096 +frame_id: '' and grasped_object_mask header seq: 453444 +stamp: + secs: 1733933432 + nsecs: 261237860 +frame_id: "camera_color_optical_frame" +[INFO] [1733933432.682483]: received mask_obs with header seq: 19015 +stamp: + secs: 1733933432 + nsecs: 683989048 +frame_id: '' and grasped_object_mask header seq: 453450 +stamp: + secs: 1733933432 + nsecs: 460837603 +frame_id: "camera_color_optical_frame" +[INFO] [1733933432.881821]: received mask_obs with header seq: 19016 +stamp: + secs: 1733933432 + nsecs: 883161306 +frame_id: '' and grasped_object_mask header seq: 453456 +stamp: + secs: 1733933432 + nsecs: 660425186 +frame_id: "camera_color_optical_frame" +[INFO] [1733933433.083002]: received mask_obs with header seq: 19017 +stamp: + secs: 1733933433 + nsecs: 84469079 +frame_id: '' and grasped_object_mask header seq: 453462 +stamp: + secs: 1733933432 + nsecs: 860001802 +frame_id: "camera_color_optical_frame" +[INFO] [1733933433.283502]: received mask_obs with header seq: 19018 +stamp: + secs: 1733933433 + nsecs: 284959793 +frame_id: '' and grasped_object_mask header seq: 453468 +stamp: + secs: 1733933433 + nsecs: 59607506 +frame_id: "camera_color_optical_frame" +[INFO] [1733933433.480617]: received mask_obs with header seq: 19019 +stamp: + secs: 1733933433 + nsecs: 482108831 +frame_id: '' and grasped_object_mask header seq: 453474 +stamp: + secs: 1733933433 + nsecs: 259208202 +frame_id: "camera_color_optical_frame" +[INFO] [1733933433.691450]: received mask_obs with header seq: 19020 +stamp: + secs: 1733933433 + nsecs: 692481994 +frame_id: '' and grasped_object_mask header seq: 453480 +stamp: + secs: 1733933433 + nsecs: 458797455 +frame_id: "camera_color_optical_frame" +[INFO] [1733933433.888831]: received mask_obs with header seq: 19021 +stamp: + secs: 1733933433 + nsecs: 890350103 +frame_id: '' and grasped_object_mask header seq: 453486 +stamp: + secs: 1733933433 + nsecs: 658375740 +frame_id: "camera_color_optical_frame" +[INFO] [1733933434.082551]: received mask_obs with header seq: 19022 +stamp: + secs: 1733933434 + nsecs: 82711696 +frame_id: '' and grasped_object_mask header seq: 453492 +stamp: + secs: 1733933433 + nsecs: 857941151 +frame_id: "camera_color_optical_frame" +[INFO] [1733933434.283210]: received mask_obs with header seq: 19023 +stamp: + secs: 1733933434 + nsecs: 284617185 +frame_id: '' and grasped_object_mask header seq: 453498 +stamp: + secs: 1733933434 + nsecs: 57544231 +frame_id: "camera_color_optical_frame" +[INFO] [1733933434.483042]: received mask_obs with header seq: 19024 +stamp: + secs: 1733933434 + nsecs: 484491825 +frame_id: '' and grasped_object_mask header seq: 453504 +stamp: + secs: 1733933434 + nsecs: 257144928 +frame_id: "camera_color_optical_frame" +[INFO] [1733933434.690217]: received mask_obs with header seq: 19025 +stamp: + secs: 1733933434 + nsecs: 691742897 +frame_id: '' and grasped_object_mask header seq: 453510 +stamp: + secs: 1733933434 + nsecs: 456733942 +frame_id: "camera_color_optical_frame" +[INFO] [1733933434.887181]: received mask_obs with header seq: 19026 +stamp: + secs: 1733933434 + nsecs: 888506412 +frame_id: '' and grasped_object_mask header seq: 453516 +stamp: + secs: 1733933434 + nsecs: 656310797 +frame_id: "camera_color_optical_frame" +[INFO] [1733933435.081461]: received mask_obs with header seq: 19027 +stamp: + secs: 1733933435 + nsecs: 82660913 +frame_id: '' and grasped_object_mask header seq: 453522 +stamp: + secs: 1733933434 + nsecs: 855876923 +frame_id: "camera_color_optical_frame" +[INFO] [1733933435.282434]: received mask_obs with header seq: 19028 +stamp: + secs: 1733933435 + nsecs: 283955812 +frame_id: '' and grasped_object_mask header seq: 453528 +stamp: + secs: 1733933435 + nsecs: 55483103 +frame_id: "camera_color_optical_frame" +[INFO] [1733933435.482997]: received mask_obs with header seq: 19029 +stamp: + secs: 1733933435 + nsecs: 484386205 +frame_id: '' and grasped_object_mask header seq: 453534 +stamp: + secs: 1733933435 + nsecs: 255088568 +frame_id: "camera_color_optical_frame" +[INFO] [1733933435.682400]: received mask_obs with header seq: 19030 +stamp: + secs: 1733933435 + nsecs: 683875322 +frame_id: '' and grasped_object_mask header seq: 453540 +stamp: + secs: 1733933435 + nsecs: 454682589 +frame_id: "camera_color_optical_frame" +[INFO] [1733933435.883064]: received mask_obs with header seq: 19031 +stamp: + secs: 1733933435 + nsecs: 884474992 +frame_id: '' and grasped_object_mask header seq: 453546 +stamp: + secs: 1733933435 + nsecs: 654264927 +frame_id: "camera_color_optical_frame" +[INFO] [1733933436.082375]: received mask_obs with header seq: 19032 +stamp: + secs: 1733933436 + nsecs: 83657741 +frame_id: '' and grasped_object_mask header seq: 453552 +stamp: + secs: 1733933435 + nsecs: 853835821 +frame_id: "camera_color_optical_frame" +[INFO] [1733933436.286821]: received mask_obs with header seq: 19033 +stamp: + secs: 1733933436 + nsecs: 288077354 +frame_id: '' and grasped_object_mask header seq: 453558 +stamp: + secs: 1733933436 + nsecs: 54991245 +frame_id: "camera_color_optical_frame" +[INFO] [1733933436.485786]: received mask_obs with header seq: 19034 +stamp: + secs: 1733933436 + nsecs: 487067699 +frame_id: '' and grasped_object_mask header seq: 453564 +stamp: + secs: 1733933436 + nsecs: 256980181 +frame_id: "camera_color_optical_frame" +[INFO] [1733933436.686275]: received mask_obs with header seq: 19035 +stamp: + secs: 1733933436 + nsecs: 687690734 +frame_id: '' and grasped_object_mask header seq: 453570 +stamp: + secs: 1733933436 + nsecs: 459045887 +frame_id: "camera_color_optical_frame" +[INFO] [1733933436.892660]: received mask_obs with header seq: 19036 +stamp: + secs: 1733933436 + nsecs: 893957138 +frame_id: '' and grasped_object_mask header seq: 453576 +stamp: + secs: 1733933436 + nsecs: 661187649 +frame_id: "camera_color_optical_frame" +[INFO] [1733933437.085446]: received mask_obs with header seq: 19037 +stamp: + secs: 1733933437 + nsecs: 86939573 +frame_id: '' and grasped_object_mask header seq: 453582 +stamp: + secs: 1733933436 + nsecs: 863405943 +frame_id: "camera_color_optical_frame" +[INFO] [1733933437.281724]: received mask_obs with header seq: 19038 +stamp: + secs: 1733933437 + nsecs: 282562017 +frame_id: '' and grasped_object_mask header seq: 453588 +stamp: + secs: 1733933437 + nsecs: 63765049 +frame_id: "camera_color_optical_frame" +[INFO] [1733933437.482576]: received mask_obs with header seq: 19039 +stamp: + secs: 1733933437 + nsecs: 484067678 +frame_id: '' and grasped_object_mask header seq: 453594 +stamp: + secs: 1733933437 + nsecs: 263150692 +frame_id: "camera_color_optical_frame" +[INFO] [1733933437.679905]: received mask_obs with header seq: 19040 +stamp: + secs: 1733933437 + nsecs: 681392192 +frame_id: '' and grasped_object_mask header seq: 453600 +stamp: + secs: 1733933437 + nsecs: 462512493 +frame_id: "camera_color_optical_frame" +[INFO] [1733933437.881925]: received mask_obs with header seq: 19041 +stamp: + secs: 1733933437 + nsecs: 882842063 +frame_id: '' and grasped_object_mask header seq: 453606 +stamp: + secs: 1733933437 + nsecs: 661851168 +frame_id: "camera_color_optical_frame" +[INFO] [1733933438.081510]: received mask_obs with header seq: 19042 +stamp: + secs: 1733933438 + nsecs: 82718372 +frame_id: '' and grasped_object_mask header seq: 453612 +stamp: + secs: 1733933437 + nsecs: 861166716 +frame_id: "camera_color_optical_frame" +[INFO] [1733933438.284147]: received mask_obs with header seq: 19043 +stamp: + secs: 1733933438 + nsecs: 285467147 +frame_id: '' and grasped_object_mask header seq: 453618 +stamp: + secs: 1733933438 + nsecs: 61383724 +frame_id: "camera_color_optical_frame" +[INFO] [1733933438.484356]: received mask_obs with header seq: 19044 +stamp: + secs: 1733933438 + nsecs: 485886573 +frame_id: '' and grasped_object_mask header seq: 453624 +stamp: + secs: 1733933438 + nsecs: 262122869 +frame_id: "camera_color_optical_frame" +[INFO] [1733933438.684093]: received mask_obs with header seq: 19045 +stamp: + secs: 1733933438 + nsecs: 685458421 +frame_id: '' and grasped_object_mask header seq: 453630 +stamp: + secs: 1733933438 + nsecs: 462939262 +frame_id: "camera_color_optical_frame" +[INFO] [1733933438.884635]: received mask_obs with header seq: 19046 +stamp: + secs: 1733933438 + nsecs: 886133432 +frame_id: '' and grasped_object_mask header seq: 453636 +stamp: + secs: 1733933438 + nsecs: 663831234 +frame_id: "camera_color_optical_frame" +[INFO] [1733933439.084817]: received mask_obs with header seq: 19047 +stamp: + secs: 1733933439 + nsecs: 86287021 +frame_id: '' and grasped_object_mask header seq: 453642 +stamp: + secs: 1733933438 + nsecs: 864799500 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -142322883 for nearest: 1733933438922564164 - target: 1733933439064887047 at idx 138 +[INFO] [1733933439.284725]: received mask_obs with header seq: 19048 +stamp: + secs: 1733933439 + nsecs: 285979986 +frame_id: '' and grasped_object_mask header seq: 453648 +stamp: + secs: 1733933439 + nsecs: 64887047 +frame_id: "camera_color_optical_frame" +[INFO] [1733933439.483143]: received mask_obs with header seq: 19049 +stamp: + secs: 1733933439 + nsecs: 484560728 +frame_id: '' and grasped_object_mask header seq: 453654 +stamp: + secs: 1733933439 + nsecs: 264498949 +frame_id: "camera_color_optical_frame" +[INFO] [1733933439.683968]: received mask_obs with header seq: 19050 +stamp: + secs: 1733933439 + nsecs: 685111284 +frame_id: '' and grasped_object_mask header seq: 453660 +stamp: + secs: 1733933439 + nsecs: 464099646 +frame_id: "camera_color_optical_frame" +[INFO] [1733933439.883921]: received mask_obs with header seq: 19051 +stamp: + secs: 1733933439 + nsecs: 884912014 +frame_id: '' and grasped_object_mask header seq: 453666 +stamp: + secs: 1733933439 + nsecs: 663688660 +frame_id: "camera_color_optical_frame" +[INFO] [1733933440.083367]: received mask_obs with header seq: 19052 +stamp: + secs: 1733933440 + nsecs: 84665775 +frame_id: '' and grasped_object_mask header seq: 453672 +stamp: + secs: 1733933439 + nsecs: 863267660 +frame_id: "camera_color_optical_frame" +[INFO] [1733933440.284996]: received mask_obs with header seq: 19053 +stamp: + secs: 1733933440 + nsecs: 286109447 +frame_id: '' and grasped_object_mask header seq: 453678 +stamp: + secs: 1733933440 + nsecs: 63742638 +frame_id: "camera_color_optical_frame" +[INFO] [1733933440.483284]: received mask_obs with header seq: 19054 +stamp: + secs: 1733933440 + nsecs: 484810590 +frame_id: '' and grasped_object_mask header seq: 453684 +stamp: + secs: 1733933440 + nsecs: 265704632 +frame_id: "camera_color_optical_frame" +[INFO] [1733933440.694135]: received mask_obs with header seq: 19055 +stamp: + secs: 1733933440 + nsecs: 695662260 +frame_id: '' and grasped_object_mask header seq: 453690 +stamp: + secs: 1733933440 + nsecs: 467742682 +frame_id: "camera_color_optical_frame" +[INFO] [1733933440.902978]: received mask_obs with header seq: 19056 +stamp: + secs: 1733933440 + nsecs: 904166698 +frame_id: '' and grasped_object_mask header seq: 453696 +stamp: + secs: 1733933440 + nsecs: 669856787 +frame_id: "camera_color_optical_frame" +[INFO] [1733933441.082974]: received mask_obs with header seq: 19057 +stamp: + secs: 1733933441 + nsecs: 84328174 +frame_id: '' and grasped_object_mask header seq: 453702 +stamp: + secs: 1733933440 + nsecs: 872046471 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -250833394 for nearest: 1733933440822515367 - target: 1733933441073348761 at idx 162 +[INFO] [1733933441.284051]: received mask_obs with header seq: 19058 +stamp: + secs: 1733933441 + nsecs: 285051822 +frame_id: '' and grasped_object_mask header seq: 453708 +stamp: + secs: 1733933441 + nsecs: 73348761 +frame_id: "camera_color_optical_frame" +[INFO] [1733933441.482312]: received mask_obs with header seq: 19059 +stamp: + secs: 1733933441 + nsecs: 483774423 +frame_id: '' and grasped_object_mask header seq: 453714 +stamp: + secs: 1733933441 + nsecs: 272731304 +frame_id: "camera_color_optical_frame" +[INFO] [1733933441.685155]: received mask_obs with header seq: 19060 +stamp: + secs: 1733933441 + nsecs: 686502695 +frame_id: '' and grasped_object_mask header seq: 453720 +stamp: + secs: 1733933441 + nsecs: 472090244 +frame_id: "camera_color_optical_frame" +[INFO] [1733933441.881785]: received mask_obs with header seq: 19061 +stamp: + secs: 1733933441 + nsecs: 882995367 +frame_id: '' and grasped_object_mask header seq: 453726 +stamp: + secs: 1733933441 + nsecs: 671426296 +frame_id: "camera_color_optical_frame" +[INFO] [1733933442.082760]: received mask_obs with header seq: 19062 +stamp: + secs: 1733933442 + nsecs: 84327459 +frame_id: '' and grasped_object_mask header seq: 453732 +stamp: + secs: 1733933441 + nsecs: 870738983 +frame_id: "camera_color_optical_frame" +[INFO] [1733933442.298901]: received mask_obs with header seq: 19063 +stamp: + secs: 1733933442 + nsecs: 300136804 +frame_id: '' and grasped_object_mask header seq: 453738 +stamp: + secs: 1733933442 + nsecs: 70062637 +frame_id: "camera_color_optical_frame" +[INFO] [1733933442.491119]: received mask_obs with header seq: 19064 +stamp: + secs: 1733933442 + nsecs: 492618083 +frame_id: '' and grasped_object_mask header seq: 453744 +stamp: + secs: 1733933442 + nsecs: 269432783 +frame_id: "camera_color_optical_frame" +[INFO] [1733933442.683114]: received mask_obs with header seq: 19065 +stamp: + secs: 1733933442 + nsecs: 684608459 +frame_id: '' and grasped_object_mask header seq: 453750 +stamp: + secs: 1733933442 + nsecs: 468778133 +frame_id: "camera_color_optical_frame" +[INFO] [1733933442.882629]: received mask_obs with header seq: 19066 +stamp: + secs: 1733933442 + nsecs: 884158372 +frame_id: '' and grasped_object_mask header seq: 453756 +stamp: + secs: 1733933442 + nsecs: 668099642 +frame_id: "camera_color_optical_frame" +[INFO] [1733933443.082695]: received mask_obs with header seq: 19067 +stamp: + secs: 1733933443 + nsecs: 84257125 +frame_id: '' and grasped_object_mask header seq: 453762 +stamp: + secs: 1733933442 + nsecs: 867397070 +frame_id: "camera_color_optical_frame" +[INFO] [1733933443.284319]: received mask_obs with header seq: 19068 +stamp: + secs: 1733933443 + nsecs: 285676479 +frame_id: '' and grasped_object_mask header seq: 453768 +stamp: + secs: 1733933443 + nsecs: 66710234 +frame_id: "camera_color_optical_frame" +[INFO] [1733933443.482048]: received mask_obs with header seq: 19069 +stamp: + secs: 1733933443 + nsecs: 482686281 +frame_id: '' and grasped_object_mask header seq: 453774 +stamp: + secs: 1733933443 + nsecs: 266078472 +frame_id: "camera_color_optical_frame" +[INFO] [1733933443.705409]: received mask_obs with header seq: 19070 +stamp: + secs: 1733933443 + nsecs: 706383466 +frame_id: '' and grasped_object_mask header seq: 453780 +stamp: + secs: 1733933443 + nsecs: 465423584 +frame_id: "camera_color_optical_frame" +[INFO] [1733933443.920511]: received mask_obs with header seq: 19071 +stamp: + secs: 1733933443 + nsecs: 921935081 +frame_id: '' and grasped_object_mask header seq: 453787 +stamp: + secs: 1733933443 + nsecs: 697961092 +frame_id: "camera_color_optical_frame" +[INFO] [1733933444.115481]: received mask_obs with header seq: 19072 +stamp: + secs: 1733933444 + nsecs: 116945981 +frame_id: '' and grasped_object_mask header seq: 453793 +stamp: + secs: 1733933443 + nsecs: 897253752 +frame_id: "camera_color_optical_frame" +[INFO] [1733933444.318207]: received mask_obs with header seq: 19073 +stamp: + secs: 1733933444 + nsecs: 319221973 +frame_id: '' and grasped_object_mask header seq: 453799 +stamp: + secs: 1733933444 + nsecs: 96584320 +frame_id: "camera_color_optical_frame" +[INFO] [1733933444.514589]: received mask_obs with header seq: 19074 +stamp: + secs: 1733933444 + nsecs: 516119956 +frame_id: '' and grasped_object_mask header seq: 453805 +stamp: + secs: 1733933444 + nsecs: 295951128 +frame_id: "camera_color_optical_frame" +[INFO] [1733933444.709912]: received mask_obs with header seq: 19075 +stamp: + secs: 1733933444 + nsecs: 711442708 +frame_id: '' and grasped_object_mask header seq: 453811 +stamp: + secs: 1733933444 + nsecs: 495294094 +frame_id: "camera_color_optical_frame" +[INFO] [1733933444.912813]: received mask_obs with header seq: 19076 +stamp: + secs: 1733933444 + nsecs: 914364814 +frame_id: '' and grasped_object_mask header seq: 453817 +stamp: + secs: 1733933444 + nsecs: 694613457 +frame_id: "camera_color_optical_frame" +[INFO] [1733933445.111553]: received mask_obs with header seq: 19077 +stamp: + secs: 1733933445 + nsecs: 113110303 +frame_id: '' and grasped_object_mask header seq: 453823 +stamp: + secs: 1733933444 + nsecs: 893908978 +frame_id: "camera_color_optical_frame" +[INFO] [1733933445.315149]: received mask_obs with header seq: 19078 +stamp: + secs: 1733933445 + nsecs: 316237449 +frame_id: '' and grasped_object_mask header seq: 453829 +stamp: + secs: 1733933445 + nsecs: 93328714 +frame_id: "camera_color_optical_frame" +[INFO] [1733933445.513666]: received mask_obs with header seq: 19079 +stamp: + secs: 1733933445 + nsecs: 515228748 +frame_id: '' and grasped_object_mask header seq: 453835 +stamp: + secs: 1733933445 + nsecs: 292724609 +frame_id: "camera_color_optical_frame" +[INFO] [1733933445.704451]: received mask_obs with header seq: 19080 +stamp: + secs: 1733933445 + nsecs: 706020593 +frame_id: '' and grasped_object_mask header seq: 453841 +stamp: + secs: 1733933445 + nsecs: 492096663 +frame_id: "camera_color_optical_frame" +[INFO] [1733933445.908691]: received mask_obs with header seq: 19081 +stamp: + secs: 1733933445 + nsecs: 909635305 +frame_id: '' and grasped_object_mask header seq: 453847 +stamp: + secs: 1733933445 + nsecs: 691446066 +frame_id: "camera_color_optical_frame" +[INFO] [1733933446.104315]: received mask_obs with header seq: 19082 +stamp: + secs: 1733933446 + nsecs: 105346441 +frame_id: '' and grasped_object_mask header seq: 453853 +stamp: + secs: 1733933445 + nsecs: 890772343 +frame_id: "camera_color_optical_frame" +[INFO] [1733933446.283918]: received mask_obs with header seq: 19083 +stamp: + secs: 1733933446 + nsecs: 285450220 +frame_id: '' and grasped_object_mask header seq: 453858 +stamp: + secs: 1733933446 + nsecs: 56877375 +frame_id: "camera_color_optical_frame" +[INFO] [1733933446.484170]: received mask_obs with header seq: 19084 +stamp: + secs: 1733933446 + nsecs: 485732793 +frame_id: '' and grasped_object_mask header seq: 453864 +stamp: + secs: 1733933446 + nsecs: 256269217 +frame_id: "camera_color_optical_frame" +[INFO] [1733933446.683475]: received mask_obs with header seq: 19085 +stamp: + secs: 1733933446 + nsecs: 684994459 +frame_id: '' and grasped_object_mask header seq: 453870 +stamp: + secs: 1733933446 + nsecs: 455638170 +frame_id: "camera_color_optical_frame" +[INFO] [1733933446.883994]: received mask_obs with header seq: 19086 +stamp: + secs: 1733933446 + nsecs: 885382413 +frame_id: '' and grasped_object_mask header seq: 453876 +stamp: + secs: 1733933446 + nsecs: 654984236 +frame_id: "camera_color_optical_frame" +[INFO] [1733933447.082270]: received mask_obs with header seq: 19087 +stamp: + secs: 1733933447 + nsecs: 83689212 +frame_id: '' and grasped_object_mask header seq: 453882 +stamp: + secs: 1733933446 + nsecs: 854307175 +frame_id: "camera_color_optical_frame" +[INFO] [1733933447.298599]: received mask_obs with header seq: 19088 +stamp: + secs: 1733933447 + nsecs: 299512863 +frame_id: '' and grasped_object_mask header seq: 453888 +stamp: + secs: 1733933447 + nsecs: 53624153 +frame_id: "camera_color_optical_frame" +[INFO] [1733933447.486137]: received mask_obs with header seq: 19089 +stamp: + secs: 1733933447 + nsecs: 487441778 +frame_id: '' and grasped_object_mask header seq: 453894 +stamp: + secs: 1733933447 + nsecs: 253005028 +frame_id: "camera_color_optical_frame" +[INFO] [1733933447.683656]: received mask_obs with header seq: 19090 +stamp: + secs: 1733933447 + nsecs: 685211658 +frame_id: '' and grasped_object_mask header seq: 453900 +stamp: + secs: 1733933447 + nsecs: 452361584 +frame_id: "camera_color_optical_frame" +[INFO] [1733933447.883610]: received mask_obs with header seq: 19091 +stamp: + secs: 1733933447 + nsecs: 885135412 +frame_id: '' and grasped_object_mask header seq: 453906 +stamp: + secs: 1733933447 + nsecs: 651694298 +frame_id: "camera_color_optical_frame" +[INFO] [1733933448.084227]: received mask_obs with header seq: 19092 +stamp: + secs: 1733933448 + nsecs: 85448026 +frame_id: '' and grasped_object_mask header seq: 453912 +stamp: + secs: 1733933447 + nsecs: 851003647 +frame_id: "camera_color_optical_frame" +[INFO] [1733933448.285613]: received mask_obs with header seq: 19093 +stamp: + secs: 1733933448 + nsecs: 287224054 +frame_id: '' and grasped_object_mask header seq: 453918 +stamp: + secs: 1733933448 + nsecs: 50309896 +frame_id: "camera_color_optical_frame" +[INFO] [1733933448.487180]: received mask_obs with header seq: 19094 +stamp: + secs: 1733933448 + nsecs: 488752365 +frame_id: '' and grasped_object_mask header seq: 453924 +stamp: + secs: 1733933448 + nsecs: 249695301 +frame_id: "camera_color_optical_frame" +[INFO] [1733933448.682187]: received mask_obs with header seq: 19095 +stamp: + secs: 1733933448 + nsecs: 683658123 +frame_id: '' and grasped_object_mask header seq: 453930 +stamp: + secs: 1733933448 + nsecs: 449058294 +frame_id: "camera_color_optical_frame" +[INFO] [1733933448.885010]: received mask_obs with header seq: 19096 +stamp: + secs: 1733933448 + nsecs: 886308193 +frame_id: '' and grasped_object_mask header seq: 453936 +stamp: + secs: 1733933448 + nsecs: 648397207 +frame_id: "camera_color_optical_frame" +[INFO] [1733933449.081166]: received mask_obs with header seq: 19097 +stamp: + secs: 1733933449 + nsecs: 82084417 +frame_id: '' and grasped_object_mask header seq: 453942 +stamp: + secs: 1733933448 + nsecs: 847712994 +frame_id: "camera_color_optical_frame" +[INFO] [1733933449.304285]: received mask_obs with header seq: 19098 +stamp: + secs: 1733933449 + nsecs: 305521726 +frame_id: '' and grasped_object_mask header seq: 453949 +stamp: + secs: 1733933449 + nsecs: 80255508 +frame_id: "camera_color_optical_frame" +[INFO] [1733933449.485013]: received mask_obs with header seq: 19099 +stamp: + secs: 1733933449 + nsecs: 486397981 +frame_id: '' and grasped_object_mask header seq: 453954 +stamp: + secs: 1733933449 + nsecs: 246403456 +frame_id: "camera_color_optical_frame" +[INFO] [1733933449.698207]: received mask_obs with header seq: 19100 +stamp: + secs: 1733933449 + nsecs: 699755430 +frame_id: '' and grasped_object_mask header seq: 453960 +stamp: + secs: 1733933449 + nsecs: 445759773 +frame_id: "camera_color_optical_frame" +[INFO] [1733933449.885347]: received mask_obs with header seq: 19101 +stamp: + secs: 1733933449 + nsecs: 886911153 +frame_id: '' and grasped_object_mask header seq: 453966 +stamp: + secs: 1733933449 + nsecs: 645093203 +frame_id: "camera_color_optical_frame" +[INFO] [1733933450.085033]: received mask_obs with header seq: 19102 +stamp: + secs: 1733933450 + nsecs: 86639404 +frame_id: '' and grasped_object_mask header seq: 453972 +stamp: + secs: 1733933449 + nsecs: 844402075 +frame_id: "camera_color_optical_frame" +[INFO] [1733933450.283572]: received mask_obs with header seq: 19103 +stamp: + secs: 1733933450 + nsecs: 285126924 +frame_id: '' and grasped_object_mask header seq: 453978 +stamp: + secs: 1733933450 + nsecs: 43702841 +frame_id: "camera_color_optical_frame" +[INFO] [1733933450.481985]: received mask_obs with header seq: 19104 +stamp: + secs: 1733933450 + nsecs: 483471870 +frame_id: '' and grasped_object_mask header seq: 453984 +stamp: + secs: 1733933450 + nsecs: 243074179 +frame_id: "camera_color_optical_frame" +[INFO] [1733933450.686333]: received mask_obs with header seq: 19105 +stamp: + secs: 1733933450 + nsecs: 687679290 +frame_id: '' and grasped_object_mask header seq: 453990 +stamp: + secs: 1733933450 + nsecs: 442421436 +frame_id: "camera_color_optical_frame" +[INFO] [1733933450.888274]: received mask_obs with header seq: 19106 +stamp: + secs: 1733933450 + nsecs: 889827251 +frame_id: '' and grasped_object_mask header seq: 453996 +stamp: + secs: 1733933450 + nsecs: 641744614 +frame_id: "camera_color_optical_frame" +[INFO] [1733933451.088908]: received mask_obs with header seq: 19107 +stamp: + secs: 1733933451 + nsecs: 90320348 +frame_id: '' and grasped_object_mask header seq: 454002 +stamp: + secs: 1733933450 + nsecs: 841044664 +frame_id: "camera_color_optical_frame" +[INFO] [1733933451.291739]: received mask_obs with header seq: 19108 +stamp: + secs: 1733933451 + nsecs: 293326616 +frame_id: '' and grasped_object_mask header seq: 454008 +stamp: + secs: 1733933451 + nsecs: 40336609 +frame_id: "camera_color_optical_frame" +[INFO] [1733933451.485448]: received mask_obs with header seq: 19109 +stamp: + secs: 1733933451 + nsecs: 486764907 +frame_id: '' and grasped_object_mask header seq: 454014 +stamp: + secs: 1733933451 + nsecs: 239708424 +frame_id: "camera_color_optical_frame" +[INFO] [1733933451.688217]: received mask_obs with header seq: 19110 +stamp: + secs: 1733933451 + nsecs: 689831256 +frame_id: '' and grasped_object_mask header seq: 454021 +stamp: + secs: 1733933451 + nsecs: 472279310 +frame_id: "camera_color_optical_frame" +[INFO] [1733933451.883533]: received mask_obs with header seq: 19111 +stamp: + secs: 1733933451 + nsecs: 885118484 +frame_id: '' and grasped_object_mask header seq: 454026 +stamp: + secs: 1733933451 + nsecs: 638380051 +frame_id: "camera_color_optical_frame" +[INFO] [1733933452.085331]: received mask_obs with header seq: 19112 +stamp: + secs: 1733933452 + nsecs: 86906194 +frame_id: '' and grasped_object_mask header seq: 454032 +stamp: + secs: 1733933451 + nsecs: 837679386 +frame_id: "camera_color_optical_frame" +[INFO] [1733933452.285238]: received mask_obs with header seq: 19113 +stamp: + secs: 1733933452 + nsecs: 286777257 +frame_id: '' and grasped_object_mask header seq: 454039 +stamp: + secs: 1733933452 + nsecs: 70586920 +frame_id: "camera_color_optical_frame" +[INFO] [1733933452.482450]: received mask_obs with header seq: 19114 +stamp: + secs: 1733933452 + nsecs: 484046697 +frame_id: '' and grasped_object_mask header seq: 454044 +stamp: + secs: 1733933452 + nsecs: 237833500 +frame_id: "camera_color_optical_frame" +[INFO] [1733933452.691013]: received mask_obs with header seq: 19115 +stamp: + secs: 1733933452 + nsecs: 692604780 +frame_id: '' and grasped_object_mask header seq: 454050 +stamp: + secs: 1733933452 + nsecs: 438598394 +frame_id: "camera_color_optical_frame" +[INFO] [1733933452.886112]: received mask_obs with header seq: 19116 +stamp: + secs: 1733933452 + nsecs: 887473583 +frame_id: '' and grasped_object_mask header seq: 454056 +stamp: + secs: 1733933452 + nsecs: 639439583 +frame_id: "camera_color_optical_frame" +[INFO] [1733933453.082456]: received mask_obs with header seq: 19117 +stamp: + secs: 1733933453 + nsecs: 83918094 +frame_id: '' and grasped_object_mask header seq: 454063 +stamp: + secs: 1733933452 + nsecs: 873851061 +frame_id: "camera_color_optical_frame" +[INFO] [1733933453.291371]: received mask_obs with header seq: 19118 +stamp: + secs: 1733933453 + nsecs: 292893886 +frame_id: '' and grasped_object_mask header seq: 454069 +stamp: + secs: 1733933453 + nsecs: 74432135 +frame_id: "camera_color_optical_frame" +[INFO] [1733933453.486279]: received mask_obs with header seq: 19119 +stamp: + secs: 1733933453 + nsecs: 487900972 +frame_id: '' and grasped_object_mask header seq: 454074 +stamp: + secs: 1733933453 + nsecs: 240777016 +frame_id: "camera_color_optical_frame" +[INFO] [1733933453.687867]: received mask_obs with header seq: 19120 +stamp: + secs: 1733933453 + nsecs: 689456224 +frame_id: '' and grasped_object_mask header seq: 454081 +stamp: + secs: 1733933453 + nsecs: 473645210 +frame_id: "camera_color_optical_frame" +[INFO] [1733933453.889925]: received mask_obs with header seq: 19121 +stamp: + secs: 1733933453 + nsecs: 891385555 +frame_id: '' and grasped_object_mask header seq: 454087 +stamp: + secs: 1733933453 + nsecs: 673234701 +frame_id: "camera_color_optical_frame" +[INFO] [1733933454.083446]: received mask_obs with header seq: 19122 +stamp: + secs: 1733933454 + nsecs: 84690093 +frame_id: '' and grasped_object_mask header seq: 454093 +stamp: + secs: 1733933453 + nsecs: 872813463 +frame_id: "camera_color_optical_frame" +[INFO] [1733933454.284905]: received mask_obs with header seq: 19123 +stamp: + secs: 1733933454 + nsecs: 286373138 +frame_id: '' and grasped_object_mask header seq: 454099 +stamp: + secs: 1733933454 + nsecs: 72395563 +frame_id: "camera_color_optical_frame" +[INFO] [1733933454.485610]: received mask_obs with header seq: 19124 +stamp: + secs: 1733933454 + nsecs: 487196683 +frame_id: '' and grasped_object_mask header seq: 454105 +stamp: + secs: 1733933454 + nsecs: 272002459 +frame_id: "camera_color_optical_frame" +[INFO] [1733933454.686972]: received mask_obs with header seq: 19125 +stamp: + secs: 1733933454 + nsecs: 688415765 +frame_id: '' and grasped_object_mask header seq: 454111 +stamp: + secs: 1733933454 + nsecs: 471597672 +frame_id: "camera_color_optical_frame" +[INFO] [1733933454.886483]: received mask_obs with header seq: 19126 +stamp: + secs: 1733933454 + nsecs: 887921810 +frame_id: '' and grasped_object_mask header seq: 454117 +stamp: + secs: 1733933454 + nsecs: 671180964 +frame_id: "camera_color_optical_frame" +[INFO] [1733933455.084938]: received mask_obs with header seq: 19127 +stamp: + secs: 1733933455 + nsecs: 86404561 +frame_id: '' and grasped_object_mask header seq: 454123 +stamp: + secs: 1733933454 + nsecs: 870752096 +frame_id: "camera_color_optical_frame" +[INFO] [1733933455.286935]: received mask_obs with header seq: 19128 +stamp: + secs: 1733933455 + nsecs: 288366079 +frame_id: '' and grasped_object_mask header seq: 454128 +stamp: + secs: 1733933455 + nsecs: 37061214 +frame_id: "camera_color_optical_frame" +[INFO] [1733933455.494375]: received mask_obs with header seq: 19129 +stamp: + secs: 1733933455 + nsecs: 493946313 +frame_id: '' and grasped_object_mask header seq: 454135 +stamp: + secs: 1733933455 + nsecs: 269940615 +frame_id: "camera_color_optical_frame" +[INFO] [1733933455.690906]: received mask_obs with header seq: 19130 +stamp: + secs: 1733933455 + nsecs: 692477226 +frame_id: '' and grasped_object_mask header seq: 454141 +stamp: + secs: 1733933455 + nsecs: 469538212 +frame_id: "camera_color_optical_frame" +[INFO] [1733933455.890346]: received mask_obs with header seq: 19131 +stamp: + secs: 1733933455 + nsecs: 891878366 +frame_id: '' and grasped_object_mask header seq: 454146 +stamp: + secs: 1733933455 + nsecs: 635860443 +frame_id: "camera_color_optical_frame" +[INFO] [1733933456.087658]: received mask_obs with header seq: 19132 +stamp: + secs: 1733933456 + nsecs: 89055299 +frame_id: '' and grasped_object_mask header seq: 454153 +stamp: + secs: 1733933455 + nsecs: 868698597 +frame_id: "camera_color_optical_frame" +[INFO] [1733933456.293305]: received mask_obs with header seq: 19133 +stamp: + secs: 1733933456 + nsecs: 293751716 +frame_id: '' and grasped_object_mask header seq: 454158 +stamp: + secs: 1733933456 + nsecs: 35166979 +frame_id: "camera_color_optical_frame" +[INFO] [1733933456.487562]: received mask_obs with header seq: 19134 +stamp: + secs: 1733933456 + nsecs: 489154577 +frame_id: '' and grasped_object_mask header seq: 454165 +stamp: + secs: 1733933456 + nsecs: 269581556 +frame_id: "camera_color_optical_frame" +[INFO] [1733933456.690499]: received mask_obs with header seq: 19135 +stamp: + secs: 1733933456 + nsecs: 692122697 +frame_id: '' and grasped_object_mask header seq: 454170 +stamp: + secs: 1733933456 + nsecs: 437094927 +frame_id: "camera_color_optical_frame" +[INFO] [1733933456.892412]: received mask_obs with header seq: 19136 +stamp: + secs: 1733933456 + nsecs: 893205404 +frame_id: '' and grasped_object_mask header seq: 454177 +stamp: + secs: 1733933456 + nsecs: 671716213 +frame_id: "camera_color_optical_frame" +[INFO] [1733933457.089099]: received mask_obs with header seq: 19137 +stamp: + secs: 1733933457 + nsecs: 90378046 +frame_id: '' and grasped_object_mask header seq: 454183 +stamp: + secs: 1733933456 + nsecs: 872915506 +frame_id: "camera_color_optical_frame" +[INFO] [1733933457.291456]: received mask_obs with header seq: 19138 +stamp: + secs: 1733933457 + nsecs: 292497873 +frame_id: '' and grasped_object_mask header seq: 454188 +stamp: + secs: 1733933457 + nsecs: 40468216 +frame_id: "camera_color_optical_frame" +[INFO] [1733933457.488971]: received mask_obs with header seq: 19139 +stamp: + secs: 1733933457 + nsecs: 490524530 +frame_id: '' and grasped_object_mask header seq: 454195 +stamp: + secs: 1733933457 + nsecs: 273619890 +frame_id: "camera_color_optical_frame" +[INFO] [1733933457.541542]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit[pub_text-7] killing on exit +[INFO] [1733933457.709853]: received mask_obs with header seq: 19140 +stamp: + secs: 1733933457 + nsecs: 691912889 +frame_id: '' and grasped_object_mask header seq: 454201 +stamp: + secs: 1733933457 + nsecs: 473465204 +frame_id: "camera_color_optical_frame" +[INFO] [1733933458.337013]: Stored episode 3. +[INFO] [1733933458.337785]: Saving video... +[INFO] [1733933458.565365]: Video saved! +Starting episode 3 +[INFO] [1733933458.597638]: resetting environment +[INFO] [1733933458.612522]: cleared current plan +[INFO] [1733933458.616285]: moving to home +[INFO] [1733933461.524389]: reached home +[INFO] [1733933461.531093]: reset action history +[INFO] [1733933464.259046]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1733933477.392702]: resetting environment +[INFO] [1733933477.403960]: cleared current plan +[INFO] [1733933477.405931]: moving to home +[INFO] [1733933478.410458]: reached home +[INFO] [1733933478.416332]: reset action history +[INFO] [1733933481.139851]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/episode_rosbags/episode_3_2024-12-11-11-11-21.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-736760.log +started roslaunch server http://158.130.50.37:34805/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [737442] +process[print_text-11]: started with pid [737443] +process[rosbag_record-12]: started with pid [737444] +started bagging! +[INFO] [1733933484.050274]: received mask_obs with header seq: 19141 +stamp: + secs: 1733933484 + nsecs: 51668643 +frame_id: '' and grasped_object_mask header seq: 454993 +stamp: + secs: 1733933483 + nsecs: 863711119 +frame_id: "camera_color_optical_frame" +[INFO] [1733933484.255821]: received mask_obs with header seq: 19142 +stamp: + secs: 1733933484 + nsecs: 257413387 +frame_id: '' and grasped_object_mask header seq: 454999 +stamp: + secs: 1733933484 + nsecs: 63292265 +frame_id: "camera_color_optical_frame" +[INFO] [1733933484.453881]: received mask_obs with header seq: 19143 +stamp: + secs: 1733933484 + nsecs: 455296277 +frame_id: '' and grasped_object_mask header seq: 455005 +stamp: + secs: 1733933484 + nsecs: 262900591 +frame_id: "camera_color_optical_frame" +[INFO] [1733933484.651438]: received mask_obs with header seq: 19144 +stamp: + secs: 1733933484 + nsecs: 652905702 +frame_id: '' and grasped_object_mask header seq: 455011 +stamp: + secs: 1733933484 + nsecs: 462502480 +frame_id: "camera_color_optical_frame" +[INFO] [1733933484.856464]: received mask_obs with header seq: 19145 +stamp: + secs: 1733933484 + nsecs: 857625246 +frame_id: '' and grasped_object_mask header seq: 455017 +stamp: + secs: 1733933484 + nsecs: 662092209 +frame_id: "camera_color_optical_frame" +[INFO] [1733933485.063922]: received mask_obs with header seq: 19146 +stamp: + secs: 1733933485 + nsecs: 65580129 +frame_id: '' and grasped_object_mask header seq: 455023 +stamp: + secs: 1733933484 + nsecs: 861671448 +frame_id: "camera_color_optical_frame" +[INFO] [1733933485.263276]: received mask_obs with header seq: 19147 +stamp: + secs: 1733933485 + nsecs: 264763593 +frame_id: '' and grasped_object_mask header seq: 455029 +stamp: + secs: 1733933485 + nsecs: 61237574 +frame_id: "camera_color_optical_frame" +[INFO] [1733933485.466968]: received mask_obs with header seq: 19148 +stamp: + secs: 1733933485 + nsecs: 467556953 +frame_id: '' and grasped_object_mask header seq: 455035 +stamp: + secs: 1733933485 + nsecs: 260843992 +frame_id: "camera_color_optical_frame" +[INFO] [1733933485.667711]: received mask_obs with header seq: 19149 +stamp: + secs: 1733933485 + nsecs: 667934656 +frame_id: '' and grasped_object_mask header seq: 455041 +stamp: + secs: 1733933485 + nsecs: 460445404 +frame_id: "camera_color_optical_frame" +[INFO] [1733933485.892644]: received mask_obs with header seq: 19150 +stamp: + secs: 1733933485 + nsecs: 866665124 +frame_id: '' and grasped_object_mask header seq: 455047 +stamp: + secs: 1733933485 + nsecs: 660035133 +frame_id: "camera_color_optical_frame" +[INFO] [1733933486.081547]: received mask_obs with header seq: 19151 +stamp: + secs: 1733933486 + nsecs: 82441806 +frame_id: '' and grasped_object_mask header seq: 455053 +stamp: + secs: 1733933485 + nsecs: 859612942 +frame_id: "camera_color_optical_frame" +[INFO] [1733933486.277714]: received mask_obs with header seq: 19152 +stamp: + secs: 1733933486 + nsecs: 279045104 +frame_id: '' and grasped_object_mask header seq: 455059 +stamp: + secs: 1733933486 + nsecs: 59180021 +frame_id: "camera_color_optical_frame" +[INFO] [1733933486.466796]: received mask_obs with header seq: 19153 +stamp: + secs: 1733933486 + nsecs: 468334674 +frame_id: '' and grasped_object_mask header seq: 455065 +stamp: + secs: 1733933486 + nsecs: 258785486 +frame_id: "camera_color_optical_frame" +[INFO] [1733933486.667778]: received mask_obs with header seq: 19154 +stamp: + secs: 1733933486 + nsecs: 668642044 +frame_id: '' and grasped_object_mask header seq: 455071 +stamp: + secs: 1733933486 + nsecs: 458385944 +frame_id: "camera_color_optical_frame" +[INFO] [1733933486.866153]: received mask_obs with header seq: 19155 +stamp: + secs: 1733933486 + nsecs: 867125511 +frame_id: '' and grasped_object_mask header seq: 455077 +stamp: + secs: 1733933486 + nsecs: 657974720 +frame_id: "camera_color_optical_frame" +[INFO] [1733933487.083544]: received mask_obs with header seq: 19156 +stamp: + secs: 1733933487 + nsecs: 85212707 +frame_id: '' and grasped_object_mask header seq: 455083 +stamp: + secs: 1733933486 + nsecs: 857551575 +frame_id: "camera_color_optical_frame" +[INFO] [1733933487.275037]: received mask_obs with header seq: 19157 +stamp: + secs: 1733933487 + nsecs: 276465654 +frame_id: '' and grasped_object_mask header seq: 455089 +stamp: + secs: 1733933487 + nsecs: 57116508 +frame_id: "camera_color_optical_frame" +[INFO] [1733933487.471528]: received mask_obs with header seq: 19158 +stamp: + secs: 1733933487 + nsecs: 471593379 +frame_id: '' and grasped_object_mask header seq: 455095 +stamp: + secs: 1733933487 + nsecs: 256728888 +frame_id: "camera_color_optical_frame" +[INFO] [1733933487.668917]: received mask_obs with header seq: 19159 +stamp: + secs: 1733933487 + nsecs: 670277833 +frame_id: '' and grasped_object_mask header seq: 455101 +stamp: + secs: 1733933487 + nsecs: 456330299 +frame_id: "camera_color_optical_frame" +[INFO] [1733933487.898806]: received mask_obs with header seq: 19160 +stamp: + secs: 1733933487 + nsecs: 899025917 +frame_id: '' and grasped_object_mask header seq: 455108 +stamp: + secs: 1733933487 + nsecs: 689182997 +frame_id: "camera_color_optical_frame" +[INFO] [1733933488.096860]: received mask_obs with header seq: 19161 +stamp: + secs: 1733933488 + nsecs: 98134994 +frame_id: '' and grasped_object_mask header seq: 455114 +stamp: + secs: 1733933487 + nsecs: 888759851 +frame_id: "camera_color_optical_frame" +[INFO] [1733933488.299024]: received mask_obs with header seq: 19162 +stamp: + secs: 1733933488 + nsecs: 298012256 +frame_id: '' and grasped_object_mask header seq: 455120 +stamp: + secs: 1733933488 + nsecs: 88323832 +frame_id: "camera_color_optical_frame" +[INFO] [1733933488.495143]: received mask_obs with header seq: 19163 +stamp: + secs: 1733933488 + nsecs: 496335983 +frame_id: '' and grasped_object_mask header seq: 455126 +stamp: + secs: 1733933488 + nsecs: 287930965 +frame_id: "camera_color_optical_frame" +[INFO] [1733933488.697456]: received mask_obs with header seq: 19164 +stamp: + secs: 1733933488 + nsecs: 698529005 +frame_id: '' and grasped_object_mask header seq: 455132 +stamp: + secs: 1733933488 + nsecs: 487526178 +frame_id: "camera_color_optical_frame" +[INFO] [1733933488.902359]: received mask_obs with header seq: 19165 +stamp: + secs: 1733933488 + nsecs: 903131484 +frame_id: '' and grasped_object_mask header seq: 455138 +stamp: + secs: 1733933488 + nsecs: 687109470 +frame_id: "camera_color_optical_frame" +[INFO] [1733933489.110074]: received mask_obs with header seq: 19166 +stamp: + secs: 1733933489 + nsecs: 111700534 +frame_id: '' and grasped_object_mask header seq: 455144 +stamp: + secs: 1733933488 + nsecs: 886680365 +frame_id: "camera_color_optical_frame" +[INFO] [1733933489.298679]: received mask_obs with header seq: 19167 +stamp: + secs: 1733933489 + nsecs: 298869609 +frame_id: '' and grasped_object_mask header seq: 455149 +stamp: + secs: 1733933489 + nsecs: 52981138 +frame_id: "camera_color_optical_frame" +[INFO] [1733933489.496845]: received mask_obs with header seq: 19168 +stamp: + secs: 1733933489 + nsecs: 498396635 +frame_id: '' and grasped_object_mask header seq: 455156 +stamp: + secs: 1733933489 + nsecs: 285855770 +frame_id: "camera_color_optical_frame" +[INFO] [1733933489.697836]: received mask_obs with header seq: 19169 +stamp: + secs: 1733933489 + nsecs: 698605537 +frame_id: '' and grasped_object_mask header seq: 455162 +stamp: + secs: 1733933489 + nsecs: 485456705 +frame_id: "camera_color_optical_frame" +[INFO] [1733933489.915863]: received mask_obs with header seq: 19170 +stamp: + secs: 1733933489 + nsecs: 917092800 +frame_id: '' and grasped_object_mask header seq: 455168 +stamp: + secs: 1733933489 + nsecs: 685046196 +frame_id: "camera_color_optical_frame" +[INFO] [1733933490.111584]: received mask_obs with header seq: 19171 +stamp: + secs: 1733933490 + nsecs: 113019466 +frame_id: '' and grasped_object_mask header seq: 455174 +stamp: + secs: 1733933489 + nsecs: 884624004 +frame_id: "camera_color_optical_frame" +[INFO] [1733933490.301489]: received mask_obs with header seq: 19172 +stamp: + secs: 1733933490 + nsecs: 302414655 +frame_id: '' and grasped_object_mask header seq: 455179 +stamp: + secs: 1733933490 + nsecs: 50930500 +frame_id: "camera_color_optical_frame" +[INFO] [1733933490.502288]: received mask_obs with header seq: 19173 +stamp: + secs: 1733933490 + nsecs: 489488124 +frame_id: '' and grasped_object_mask header seq: 455185 +stamp: + secs: 1733933490 + nsecs: 250541687 +frame_id: "camera_color_optical_frame" +[INFO] [1733933490.711697]: received mask_obs with header seq: 19174 +stamp: + secs: 1733933490 + nsecs: 696507453 +frame_id: '' and grasped_object_mask header seq: 455191 +stamp: + secs: 1733933490 + nsecs: 450149775 +frame_id: "camera_color_optical_frame" +[INFO] [1733933490.908860]: received mask_obs with header seq: 19175 +stamp: + secs: 1733933490 + nsecs: 910456418 +frame_id: '' and grasped_object_mask header seq: 455198 +stamp: + secs: 1733933490 + nsecs: 683011293 +frame_id: "camera_color_optical_frame" +[INFO] [1733933491.111071]: received mask_obs with header seq: 19176 +stamp: + secs: 1733933491 + nsecs: 112686872 +frame_id: '' and grasped_object_mask header seq: 455204 +stamp: + secs: 1733933490 + nsecs: 882594109 +frame_id: "camera_color_optical_frame" +[INFO] [1733933491.310665]: received mask_obs with header seq: 19177 +stamp: + secs: 1733933491 + nsecs: 312305212 +frame_id: '' and grasped_object_mask header seq: 455210 +stamp: + secs: 1733933491 + nsecs: 82166910 +frame_id: "camera_color_optical_frame" +[INFO] [1733933491.510437]: received mask_obs with header seq: 19178 +stamp: + secs: 1733933491 + nsecs: 511957168 +frame_id: '' and grasped_object_mask header seq: 455216 +stamp: + secs: 1733933491 + nsecs: 281778574 +frame_id: "camera_color_optical_frame" +[INFO] [1733933491.700115]: received mask_obs with header seq: 19179 +stamp: + secs: 1733933491 + nsecs: 701154708 +frame_id: '' and grasped_object_mask header seq: 455221 +stamp: + secs: 1733933491 + nsecs: 448112965 +frame_id: "camera_color_optical_frame" +[INFO] [1733933491.916430]: received mask_obs with header seq: 19180 +stamp: + secs: 1733933491 + nsecs: 917778015 +frame_id: '' and grasped_object_mask header seq: 455228 +stamp: + secs: 1733933491 + nsecs: 680967331 +frame_id: "camera_color_optical_frame" +[INFO] [1733933492.105456]: received mask_obs with header seq: 19181 +stamp: + secs: 1733933492 + nsecs: 106898784 +frame_id: '' and grasped_object_mask header seq: 455234 +stamp: + secs: 1733933491 + nsecs: 880543709 +frame_id: "camera_color_optical_frame" +[INFO] [1733933492.296236]: received mask_obs with header seq: 19182 +stamp: + secs: 1733933492 + nsecs: 296530485 +frame_id: '' and grasped_object_mask header seq: 455239 +stamp: + secs: 1733933492 + nsecs: 46847820 +frame_id: "camera_color_optical_frame" +[INFO] [1733933492.501119]: received mask_obs with header seq: 19183 +stamp: + secs: 1733933492 + nsecs: 502068758 +frame_id: '' and grasped_object_mask header seq: 455246 +stamp: + secs: 1733933492 + nsecs: 279719830 +frame_id: "camera_color_optical_frame" +[INFO] [1733933492.710238]: received mask_obs with header seq: 19184 +stamp: + secs: 1733933492 + nsecs: 698827028 +frame_id: '' and grasped_object_mask header seq: 455252 +stamp: + secs: 1733933492 + nsecs: 479317188 +frame_id: "camera_color_optical_frame" +[INFO] [1733933492.899271]: received mask_obs with header seq: 19185 +stamp: + secs: 1733933492 + nsecs: 900805234 +frame_id: '' and grasped_object_mask header seq: 455258 +stamp: + secs: 1733933492 + nsecs: 678902864 +frame_id: "camera_color_optical_frame" +[INFO] [1733933493.081732]: received mask_obs with header seq: 19186 +stamp: + secs: 1733933493 + nsecs: 83404541 +frame_id: '' and grasped_object_mask header seq: 455263 +stamp: + secs: 1733933492 + nsecs: 845215797 +frame_id: "camera_color_optical_frame" +[INFO] [1733933493.276806]: received mask_obs with header seq: 19187 +stamp: + secs: 1733933493 + nsecs: 278556585 +frame_id: '' and grasped_object_mask header seq: 455269 +stamp: + secs: 1733933493 + nsecs: 44780016 +frame_id: "camera_color_optical_frame" +[INFO] [1733933493.469818]: received mask_obs with header seq: 19188 +stamp: + secs: 1733933493 + nsecs: 471358776 +frame_id: '' and grasped_object_mask header seq: 455275 +stamp: + secs: 1733933493 + nsecs: 244386196 +frame_id: "camera_color_optical_frame" +[INFO] [1733933493.663131]: received mask_obs with header seq: 19189 +stamp: + secs: 1733933493 + nsecs: 664809465 +frame_id: '' and grasped_object_mask header seq: 455281 +stamp: + secs: 1733933493 + nsecs: 443992376 +frame_id: "camera_color_optical_frame" +[INFO] [1733933493.861049]: received mask_obs with header seq: 19190 +stamp: + secs: 1733933493 + nsecs: 862129211 +frame_id: '' and grasped_object_mask header seq: 455287 +stamp: + secs: 1733933493 + nsecs: 643586159 +frame_id: "camera_color_optical_frame" +[INFO] [1733933494.060437]: received mask_obs with header seq: 19191 +stamp: + secs: 1733933494 + nsecs: 62102794 +frame_id: '' and grasped_object_mask header seq: 455293 +stamp: + secs: 1733933493 + nsecs: 843168497 +frame_id: "camera_color_optical_frame" +[INFO] [1733933494.277309]: received mask_obs with header seq: 19192 +stamp: + secs: 1733933494 + nsecs: 266969919 +frame_id: '' and grasped_object_mask header seq: 455299 +stamp: + secs: 1733933494 + nsecs: 42738676 +frame_id: "camera_color_optical_frame" +[INFO] [1733933494.464141]: received mask_obs with header seq: 19193 +stamp: + secs: 1733933494 + nsecs: 465555667 +frame_id: '' and grasped_object_mask header seq: 455305 +stamp: + secs: 1733933494 + nsecs: 242347002 +frame_id: "camera_color_optical_frame" +[INFO] [1733933494.660708]: received mask_obs with header seq: 19194 +stamp: + secs: 1733933494 + nsecs: 662331581 +frame_id: '' and grasped_object_mask header seq: 455311 +stamp: + secs: 1733933494 + nsecs: 441952944 +frame_id: "camera_color_optical_frame" +[INFO] [1733933494.875135]: received mask_obs with header seq: 19195 +stamp: + secs: 1733933494 + nsecs: 871973276 +frame_id: '' and grasped_object_mask header seq: 455317 +stamp: + secs: 1733933494 + nsecs: 641548157 +frame_id: "camera_color_optical_frame" +[INFO] [1733933495.070730]: received mask_obs with header seq: 19196 +stamp: + secs: 1733933495 + nsecs: 71882486 +frame_id: '' and grasped_object_mask header seq: 455323 +stamp: + secs: 1733933494 + nsecs: 841130972 +frame_id: "camera_color_optical_frame" +[INFO] [1733933495.263652]: received mask_obs with header seq: 19197 +stamp: + secs: 1733933495 + nsecs: 265368938 +frame_id: '' and grasped_object_mask header seq: 455329 +stamp: + secs: 1733933495 + nsecs: 40701628 +frame_id: "camera_color_optical_frame" +[INFO] [1733933495.461622]: received mask_obs with header seq: 19198 +stamp: + secs: 1733933495 + nsecs: 462267875 +frame_id: '' and grasped_object_mask header seq: 455335 +stamp: + secs: 1733933495 + nsecs: 240302563 +frame_id: "camera_color_optical_frame" +[INFO] [1733933495.663121]: received mask_obs with header seq: 19199 +stamp: + secs: 1733933495 + nsecs: 664590358 +frame_id: '' and grasped_object_mask header seq: 455341 +stamp: + secs: 1733933495 + nsecs: 439899445 +frame_id: "camera_color_optical_frame" +[INFO] [1733933495.869455]: received mask_obs with header seq: 19200 +stamp: + secs: 1733933495 + nsecs: 870405435 +frame_id: '' and grasped_object_mask header seq: 455347 +stamp: + secs: 1733933495 + nsecs: 639484406 +frame_id: "camera_color_optical_frame" +[INFO] [1733933496.076675]: received mask_obs with header seq: 19201 +stamp: + secs: 1733933496 + nsecs: 78321933 +frame_id: '' and grasped_object_mask header seq: 455353 +stamp: + secs: 1733933495 + nsecs: 839058161 +frame_id: "camera_color_optical_frame" +[INFO] [1733933496.287493]: received mask_obs with header seq: 19202 +stamp: + secs: 1733933496 + nsecs: 265295028 +frame_id: '' and grasped_object_mask header seq: 455359 +stamp: + secs: 1733933496 + nsecs: 38618565 +frame_id: "camera_color_optical_frame" +[INFO] [1733933496.488586]: received mask_obs with header seq: 19203 +stamp: + secs: 1733933496 + nsecs: 490185737 +frame_id: '' and grasped_object_mask header seq: 455366 +stamp: + secs: 1733933496 + nsecs: 271488190 +frame_id: "camera_color_optical_frame" +[INFO] [1733933496.708713]: received mask_obs with header seq: 19204 +stamp: + secs: 1733933496 + nsecs: 710291385 +frame_id: '' and grasped_object_mask header seq: 455372 +stamp: + secs: 1733933496 + nsecs: 471088409 +frame_id: "camera_color_optical_frame" +[INFO] [1733933496.920959]: received mask_obs with header seq: 19205 +stamp: + secs: 1733933496 + nsecs: 891718387 +frame_id: '' and grasped_object_mask header seq: 455378 +stamp: + secs: 1733933496 + nsecs: 670675278 +frame_id: "camera_color_optical_frame" +[INFO] [1733933497.127053]: received mask_obs with header seq: 19206 +stamp: + secs: 1733933497 + nsecs: 128560781 +frame_id: '' and grasped_object_mask header seq: 455385 +stamp: + secs: 1733933496 + nsecs: 903511286 +frame_id: "camera_color_optical_frame" +[INFO] [1733933497.326502]: received mask_obs with header seq: 19207 +stamp: + secs: 1733933497 + nsecs: 328235626 +frame_id: '' and grasped_object_mask header seq: 455391 +stamp: + secs: 1733933497 + nsecs: 103081942 +frame_id: "camera_color_optical_frame" +[INFO] [1733933497.510674]: received mask_obs with header seq: 19208 +stamp: + secs: 1733933497 + nsecs: 512084245 +frame_id: '' and grasped_object_mask header seq: 455396 +stamp: + secs: 1733933497 + nsecs: 269417763 +frame_id: "camera_color_optical_frame" +[INFO] [1733933497.719857]: received mask_obs with header seq: 19209 +stamp: + secs: 1733933497 + nsecs: 699973344 +frame_id: '' and grasped_object_mask header seq: 455402 +stamp: + secs: 1733933497 + nsecs: 469009876 +frame_id: "camera_color_optical_frame" +[INFO] [1733933497.924268]: received mask_obs with header seq: 19210 +stamp: + secs: 1733933497 + nsecs: 922872066 +frame_id: '' and grasped_object_mask header seq: 455409 +stamp: + secs: 1733933497 + nsecs: 701852083 +frame_id: "camera_color_optical_frame" +[INFO] [1733933498.127588]: received mask_obs with header seq: 19211 +stamp: + secs: 1733933498 + nsecs: 128830432 +frame_id: '' and grasped_object_mask header seq: 455415 +stamp: + secs: 1733933497 + nsecs: 901417732 +frame_id: "camera_color_optical_frame" +[INFO] [1733933498.327932]: received mask_obs with header seq: 19212 +stamp: + secs: 1733933498 + nsecs: 329685926 +frame_id: '' and grasped_object_mask header seq: 455420 +stamp: + secs: 1733933498 + nsecs: 67713261 +frame_id: "camera_color_optical_frame" +[INFO] [1733933498.528487]: received mask_obs with header seq: 19213 +stamp: + secs: 1733933498 + nsecs: 529861211 +frame_id: '' and grasped_object_mask header seq: 455427 +stamp: + secs: 1733933498 + nsecs: 302122116 +frame_id: "camera_color_optical_frame" +[INFO] [1733933498.729570]: received mask_obs with header seq: 19214 +stamp: + secs: 1733933498 + nsecs: 730780839 +frame_id: '' and grasped_object_mask header seq: 455433 +stamp: + secs: 1733933498 + nsecs: 503139257 +frame_id: "camera_color_optical_frame" +[INFO] [1733933498.911927]: received mask_obs with header seq: 19215 +stamp: + secs: 1733933498 + nsecs: 913653135 +frame_id: '' and grasped_object_mask header seq: 455438 +stamp: + secs: 1733933498 + nsecs: 670720577 +frame_id: "camera_color_optical_frame" +[INFO] [1733933499.107279]: received mask_obs with header seq: 19216 +stamp: + secs: 1733933499 + nsecs: 108546257 +frame_id: '' and grasped_object_mask header seq: 455444 +stamp: + secs: 1733933498 + nsecs: 871899605 +frame_id: "camera_color_optical_frame" +[INFO] [1733933499.294301]: received mask_obs with header seq: 19217 +stamp: + secs: 1733933499 + nsecs: 295686483 +frame_id: '' and grasped_object_mask header seq: 455450 +stamp: + secs: 1733933499 + nsecs: 73168516 +frame_id: "camera_color_optical_frame" +[INFO] [1733933499.535087]: received mask_obs with header seq: 19218 +stamp: + secs: 1733933499 + nsecs: 522902727 +frame_id: '' and grasped_object_mask header seq: 455456 +stamp: + secs: 1733933499 + nsecs: 273011684 +frame_id: "camera_color_optical_frame" +[INFO] [1733933499.737874]: received mask_obs with header seq: 19219 +stamp: + secs: 1733933499 + nsecs: 739600181 +frame_id: '' and grasped_object_mask header seq: 455463 +stamp: + secs: 1733933499 + nsecs: 506163120 +frame_id: "camera_color_optical_frame" +[INFO] [1733933499.932100]: received mask_obs with header seq: 19220 +stamp: + secs: 1733933499 + nsecs: 933726787 +frame_id: '' and grasped_object_mask header seq: 455469 +stamp: + secs: 1733933499 + nsecs: 706007242 +frame_id: "camera_color_optical_frame" +[INFO] [1733933500.158588]: received mask_obs with header seq: 19221 +stamp: + secs: 1733933500 + nsecs: 160214424 +frame_id: '' and grasped_object_mask header seq: 455475 +stamp: + secs: 1733933499 + nsecs: 905851841 +frame_id: "camera_color_optical_frame" +[INFO] [1733933500.374324]: received mask_obs with header seq: 19222 +stamp: + secs: 1733933500 + nsecs: 376002788 +frame_id: '' and grasped_object_mask header seq: 455482 +stamp: + secs: 1733933500 + nsecs: 139002323 +frame_id: "camera_color_optical_frame" +[INFO] [1733933500.568997]: received mask_obs with header seq: 19223 +stamp: + secs: 1733933500 + nsecs: 570763587 +frame_id: '' and grasped_object_mask header seq: 455488 +stamp: + secs: 1733933500 + nsecs: 338840246 +frame_id: "camera_color_optical_frame" +[INFO] [1733933500.770338]: received mask_obs with header seq: 19224 +stamp: + secs: 1733933500 + nsecs: 762589454 +frame_id: '' and grasped_object_mask header seq: 455494 +stamp: + secs: 1733933500 + nsecs: 538678169 +frame_id: "camera_color_optical_frame" +[INFO] [1733933500.957092]: received mask_obs with header seq: 19225 +stamp: + secs: 1733933500 + nsecs: 958565950 +frame_id: '' and grasped_object_mask header seq: 455500 +stamp: + secs: 1733933500 + nsecs: 738516092 +frame_id: "camera_color_optical_frame" +[INFO] [1733933501.168880]: received mask_obs with header seq: 19226 +stamp: + secs: 1733933501 + nsecs: 170474052 +frame_id: '' and grasped_object_mask header seq: 455506 +stamp: + secs: 1733933500 + nsecs: 938354254 +frame_id: "camera_color_optical_frame" +[INFO] [1733933501.365854]: received mask_obs with header seq: 19227 +stamp: + secs: 1733933501 + nsecs: 366806268 +frame_id: '' and grasped_object_mask header seq: 455512 +stamp: + secs: 1733933501 + nsecs: 138191223 +frame_id: "camera_color_optical_frame" +[INFO] [1733933501.584842]: received mask_obs with header seq: 19228 +stamp: + secs: 1733933501 + nsecs: 554022312 +frame_id: '' and grasped_object_mask header seq: 455517 +stamp: + secs: 1733933501 + nsecs: 304720640 +frame_id: "camera_color_optical_frame" +[INFO] [1733933501.787121]: received mask_obs with header seq: 19229 +stamp: + secs: 1733933501 + nsecs: 788469791 +frame_id: '' and grasped_object_mask header seq: 455525 +stamp: + secs: 1733933501 + nsecs: 571166992 +frame_id: "camera_color_optical_frame" +[INFO] [1733933501.988286]: received mask_obs with header seq: 19230 +stamp: + secs: 1733933501 + nsecs: 989524841 +frame_id: '' and grasped_object_mask header seq: 455531 +stamp: + secs: 1733933501 + nsecs: 771001816 +frame_id: "camera_color_optical_frame" +[INFO] [1733933502.190502]: received mask_obs with header seq: 19231 +stamp: + secs: 1733933502 + nsecs: 191974639 +frame_id: '' and grasped_object_mask header seq: 455537 +stamp: + secs: 1733933501 + nsecs: 970837116 +frame_id: "camera_color_optical_frame" +[INFO] [1733933502.390449]: received mask_obs with header seq: 19232 +stamp: + secs: 1733933502 + nsecs: 392075538 +frame_id: '' and grasped_object_mask header seq: 455543 +stamp: + secs: 1733933502 + nsecs: 170672417 +frame_id: "camera_color_optical_frame" +[INFO] [1733933502.587503]: received mask_obs with header seq: 19233 +stamp: + secs: 1733933502 + nsecs: 589066743 +frame_id: '' and grasped_object_mask header seq: 455549 +stamp: + secs: 1733933502 + nsecs: 370508194 +frame_id: "camera_color_optical_frame" +[INFO] [1733933502.787661]: received mask_obs with header seq: 19234 +stamp: + secs: 1733933502 + nsecs: 789447069 +frame_id: '' and grasped_object_mask header seq: 455555 +stamp: + secs: 1733933502 + nsecs: 570344210 +frame_id: "camera_color_optical_frame" +[INFO] [1733933502.995450]: received mask_obs with header seq: 19235 +stamp: + secs: 1733933502 + nsecs: 995997667 +frame_id: '' and grasped_object_mask header seq: 455561 +stamp: + secs: 1733933502 + nsecs: 770179987 +frame_id: "camera_color_optical_frame" +[INFO] [1733933503.193623]: received mask_obs with header seq: 19236 +stamp: + secs: 1733933503 + nsecs: 195268154 +frame_id: '' and grasped_object_mask header seq: 455567 +stamp: + secs: 1733933502 + nsecs: 970015764 +frame_id: "camera_color_optical_frame" +[INFO] [1733933503.391646]: received mask_obs with header seq: 19237 +stamp: + secs: 1733933503 + nsecs: 387123823 +frame_id: '' and grasped_object_mask header seq: 455572 +stamp: + secs: 1733933503 + nsecs: 136546135 +frame_id: "camera_color_optical_frame" +[INFO] [1733933503.584678]: received mask_obs with header seq: 19238 +stamp: + secs: 1733933503 + nsecs: 585500717 +frame_id: '' and grasped_object_mask header seq: 455579 +stamp: + secs: 1733933503 + nsecs: 369688988 +frame_id: "camera_color_optical_frame" +[INFO] [1733933503.808154]: received mask_obs with header seq: 19239 +stamp: + secs: 1733933503 + nsecs: 787250757 +frame_id: '' and grasped_object_mask header seq: 455585 +stamp: + secs: 1733933503 + nsecs: 569525003 +frame_id: "camera_color_optical_frame" +[INFO] [1733933504.004199]: received mask_obs with header seq: 19240 +stamp: + secs: 1733933504 + nsecs: 5926132 +frame_id: '' and grasped_object_mask header seq: 455591 +stamp: + secs: 1733933503 + nsecs: 769361019 +frame_id: "camera_color_optical_frame" +[INFO] [1733933504.187581]: received mask_obs with header seq: 19241 +stamp: + secs: 1733933504 + nsecs: 189343690 +frame_id: '' and grasped_object_mask header seq: 455597 +stamp: + secs: 1733933503 + nsecs: 969196796 +frame_id: "camera_color_optical_frame" +[INFO] [1733933504.405426]: received mask_obs with header seq: 19242 +stamp: + secs: 1733933504 + nsecs: 406983137 +frame_id: '' and grasped_object_mask header seq: 455603 +stamp: + secs: 1733933504 + nsecs: 169034004 +frame_id: "camera_color_optical_frame" +[INFO] [1733933504.614492]: received mask_obs with header seq: 19243 +stamp: + secs: 1733933504 + nsecs: 598079204 +frame_id: '' and grasped_object_mask header seq: 455609 +stamp: + secs: 1733933504 + nsecs: 368872643 +frame_id: "camera_color_optical_frame" +[INFO] [1733933504.793585]: received mask_obs with header seq: 19244 +stamp: + secs: 1733933504 + nsecs: 795168161 +frame_id: '' and grasped_object_mask header seq: 455615 +stamp: + secs: 1733933504 + nsecs: 568711996 +frame_id: "camera_color_optical_frame" +[INFO] [1733933504.997670]: received mask_obs with header seq: 19245 +stamp: + secs: 1733933504 + nsecs: 998618602 +frame_id: '' and grasped_object_mask header seq: 455621 +stamp: + secs: 1733933504 + nsecs: 768550396 +frame_id: "camera_color_optical_frame" +[INFO] [1733933505.184597]: received mask_obs with header seq: 19246 +stamp: + secs: 1733933505 + nsecs: 185940742 +frame_id: '' and grasped_object_mask header seq: 455626 +stamp: + secs: 1733933504 + nsecs: 935081720 +frame_id: "camera_color_optical_frame" +[INFO] [1733933505.390331]: received mask_obs with header seq: 19247 +stamp: + secs: 1733933505 + nsecs: 392110586 +frame_id: '' and grasped_object_mask header seq: 455633 +stamp: + secs: 1733933505 + nsecs: 168229342 +frame_id: "camera_color_optical_frame" +[INFO] [1733933505.577415]: received mask_obs with header seq: 19248 +stamp: + secs: 1733933505 + nsecs: 579145431 +frame_id: '' and grasped_object_mask header seq: 455638 +stamp: + secs: 1733933505 + nsecs: 334765434 +frame_id: "camera_color_optical_frame" +[INFO] [1733933505.806958]: received mask_obs with header seq: 19249 +stamp: + secs: 1733933505 + nsecs: 785337448 +frame_id: '' and grasped_object_mask header seq: 455645 +stamp: + secs: 1733933505 + nsecs: 567914963 +frame_id: "camera_color_optical_frame" +[INFO] [1733933506.015199]: received mask_obs with header seq: 19250 +stamp: + secs: 1733933506 + nsecs: 16399621 +frame_id: '' and grasped_object_mask header seq: 455651 +stamp: + secs: 1733933505 + nsecs: 767759323 +frame_id: "camera_color_optical_frame" +[INFO] [1733933506.220448]: received mask_obs with header seq: 19251 +stamp: + secs: 1733933506 + nsecs: 222186088 +frame_id: '' and grasped_object_mask header seq: 455658 +stamp: + secs: 1733933506 + nsecs: 909328 +frame_id: "camera_color_optical_frame" +[INFO] [1733933506.416677]: received mask_obs with header seq: 19252 +stamp: + secs: 1733933506 + nsecs: 417005300 +frame_id: '' and grasped_object_mask header seq: 455664 +stamp: + secs: 1733933506 + nsecs: 200751066 +frame_id: "camera_color_optical_frame" +[INFO] [1733933506.612936]: received mask_obs with header seq: 19253 +stamp: + secs: 1733933506 + nsecs: 614230871 +frame_id: '' and grasped_object_mask header seq: 455670 +stamp: + secs: 1733933506 + nsecs: 400592566 +frame_id: "camera_color_optical_frame" +[INFO] [1733933506.817968]: received mask_obs with header seq: 19254 +stamp: + secs: 1733933506 + nsecs: 819499015 +frame_id: '' and grasped_object_mask header seq: 455676 +stamp: + secs: 1733933506 + nsecs: 600433111 +frame_id: "camera_color_optical_frame" +[INFO] [1733933507.021111]: received mask_obs with header seq: 19255 +stamp: + secs: 1733933507 + nsecs: 22340536 +frame_id: '' and grasped_object_mask header seq: 455682 +stamp: + secs: 1733933506 + nsecs: 800274134 +frame_id: "camera_color_optical_frame" +[INFO] [1733933507.221246]: received mask_obs with header seq: 19256 +stamp: + secs: 1733933507 + nsecs: 222964286 +frame_id: '' and grasped_object_mask header seq: 455688 +stamp: + secs: 1733933507 + nsecs: 115156 +frame_id: "camera_color_optical_frame" +[INFO] [1733933507.432327]: received mask_obs with header seq: 19257 +stamp: + secs: 1733933507 + nsecs: 418201684 +frame_id: '' and grasped_object_mask header seq: 455694 +stamp: + secs: 1733933507 + nsecs: 199955463 +frame_id: "camera_color_optical_frame" +[INFO] [1733933507.630716]: received mask_obs with header seq: 19258 +stamp: + secs: 1733933507 + nsecs: 632496833 +frame_id: '' and grasped_object_mask header seq: 455700 +stamp: + secs: 1733933507 + nsecs: 399795294 +frame_id: "camera_color_optical_frame" +[INFO] [1733933507.827960]: received mask_obs with header seq: 19259 +stamp: + secs: 1733933507 + nsecs: 829017639 +frame_id: '' and grasped_object_mask header seq: 455706 +stamp: + secs: 1733933507 + nsecs: 599636078 +frame_id: "camera_color_optical_frame" +[INFO] [1733933508.027234]: received mask_obs with header seq: 19260 +stamp: + secs: 1733933508 + nsecs: 28415918 +frame_id: '' and grasped_object_mask header seq: 455712 +stamp: + secs: 1733933507 + nsecs: 799475908 +frame_id: "camera_color_optical_frame" +[INFO] [1733933508.238889]: received mask_obs with header seq: 19261 +stamp: + secs: 1733933508 + nsecs: 222607612 +frame_id: '' and grasped_object_mask header seq: 455718 +stamp: + secs: 1733933507 + nsecs: 999315739 +frame_id: "camera_color_optical_frame" +[INFO] [1733933508.468098]: received mask_obs with header seq: 19262 +stamp: + secs: 1733933508 + nsecs: 448718547 +frame_id: '' and grasped_object_mask header seq: 455724 +stamp: + secs: 1733933508 + nsecs: 199154854 +frame_id: "camera_color_optical_frame" +[INFO] [1733933508.686033]: received mask_obs with header seq: 19263 +stamp: + secs: 1733933508 + nsecs: 687251091 +frame_id: '' and grasped_object_mask header seq: 455731 +stamp: + secs: 1733933508 + nsecs: 432300091 +frame_id: "camera_color_optical_frame" +[INFO] [1733933508.894113]: received mask_obs with header seq: 19264 +stamp: + secs: 1733933508 + nsecs: 883328437 +frame_id: '' and grasped_object_mask header seq: 455738 +stamp: + secs: 1733933508 + nsecs: 665444374 +frame_id: "camera_color_optical_frame" +[INFO] [1733933509.096493]: received mask_obs with header seq: 19265 +stamp: + secs: 1733933509 + nsecs: 97203254 +frame_id: '' and grasped_object_mask header seq: 455744 +stamp: + secs: 1733933508 + nsecs: 865283489 +frame_id: "camera_color_optical_frame" +[INFO] [1733933509.285359]: received mask_obs with header seq: 19266 +stamp: + secs: 1733933509 + nsecs: 287142753 +frame_id: '' and grasped_object_mask header seq: 455749 +stamp: + secs: 1733933509 + nsecs: 31814814 +frame_id: "camera_color_optical_frame" +[INFO] [1733933509.485931]: received mask_obs with header seq: 19267 +stamp: + secs: 1733933509 + nsecs: 487467527 +frame_id: '' and grasped_object_mask header seq: 455756 +stamp: + secs: 1733933509 + nsecs: 264960289 +frame_id: "camera_color_optical_frame" +[INFO] [1733933509.703323]: received mask_obs with header seq: 19268 +stamp: + secs: 1733933509 + nsecs: 687484502 +frame_id: '' and grasped_object_mask header seq: 455762 +stamp: + secs: 1733933509 + nsecs: 464798689 +frame_id: "camera_color_optical_frame" +[INFO] [1733933509.905937]: received mask_obs with header seq: 19269 +stamp: + secs: 1733933509 + nsecs: 907700300 +frame_id: '' and grasped_object_mask header seq: 455768 +stamp: + secs: 1733933509 + nsecs: 664637089 +frame_id: "camera_color_optical_frame" +[INFO] [1733933510.110958]: received mask_obs with header seq: 19270 +stamp: + secs: 1733933510 + nsecs: 112698078 +frame_id: '' and grasped_object_mask header seq: 455775 +stamp: + secs: 1733933509 + nsecs: 897782803 +frame_id: "camera_color_optical_frame" +[INFO] [1733933510.314035]: received mask_obs with header seq: 19271 +stamp: + secs: 1733933510 + nsecs: 315684556 +frame_id: '' and grasped_object_mask header seq: 455781 +stamp: + secs: 1733933510 + nsecs: 97621441 +frame_id: "camera_color_optical_frame" +[INFO] [1733933510.515042]: received mask_obs with header seq: 19272 +stamp: + secs: 1733933510 + nsecs: 514287948 +frame_id: '' and grasped_object_mask header seq: 455787 +stamp: + secs: 1733933510 + nsecs: 299372911 +frame_id: "camera_color_optical_frame" +[INFO] [1733933510.718179]: received mask_obs with header seq: 19273 +stamp: + secs: 1733933510 + nsecs: 719949722 +frame_id: '' and grasped_object_mask header seq: 455793 +stamp: + secs: 1733933510 + nsecs: 501586676 +frame_id: "camera_color_optical_frame" +[INFO] [1733933510.918134]: received mask_obs with header seq: 19274 +stamp: + secs: 1733933510 + nsecs: 919469833 +frame_id: '' and grasped_object_mask header seq: 455799 +stamp: + secs: 1733933510 + nsecs: 703889132 +frame_id: "camera_color_optical_frame" +[INFO] [1733933511.122578]: received mask_obs with header seq: 19275 +stamp: + secs: 1733933511 + nsecs: 123041391 +frame_id: '' and grasped_object_mask header seq: 455805 +stamp: + secs: 1733933510 + nsecs: 906278133 +frame_id: "camera_color_optical_frame" +[INFO] [1733933511.183320]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1733933511.326109]: received mask_obs with header seq: 19276 +stamp: + secs: 1733933511 + nsecs: 324147701 +frame_id: '' and grasped_object_mask header seq: 455811 +stamp: + secs: 1733933511 + nsecs: 108754158 +frame_id: "camera_color_optical_frame" +[INFO] [1733933512.028825]: Stored episode 4. +[INFO] [1733933512.029758]: Saving video... +[INFO] [1733933512.289492]: Video saved! +Starting episode 4 +[INFO] [1733933512.340260]: resetting environment +[INFO] [1733933512.362681]: cleared current plan +[INFO] [1733933512.366985]: moving to home +[INFO] [1733933515.174255]: reached home +[INFO] [1733933515.177851]: reset action history +[INFO] [1733933517.962101]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1733933521.649014]: resetting environment +[INFO] [1733933521.664286]: cleared current plan +[INFO] [1733933521.664994]: moving to home +[INFO] [1733933522.665834]: reached home +[INFO] [1733933522.676108]: reset action history +[INFO] [1733933525.466357]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/episode_rosbags/episode_4_2024-12-11-11-12-05.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-736760.log +started roslaunch server http://158.130.50.37:34785/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [737577] +process[print_text-14]: started with pid [737578] +process[rosbag_record-15]: started with pid [737579] +started bagging! +[INFO] [1733933528.956224]: received mask_obs with header seq: 19277 +stamp: + secs: 1733933528 + nsecs: 957888603 +frame_id: '' and grasped_object_mask header seq: 456341 +stamp: + secs: 1733933528 + nsecs: 749752283 +frame_id: "camera_color_optical_frame" +[INFO] [1733933529.154292]: received mask_obs with header seq: 19278 +stamp: + secs: 1733933529 + nsecs: 156111478 +frame_id: '' and grasped_object_mask header seq: 456347 +stamp: + secs: 1733933528 + nsecs: 949342489 +frame_id: "camera_color_optical_frame" +[INFO] [1733933529.355096]: received mask_obs with header seq: 19279 +stamp: + secs: 1733933529 + nsecs: 356653213 +frame_id: '' and grasped_object_mask header seq: 456353 +stamp: + secs: 1733933529 + nsecs: 148922443 +frame_id: "camera_color_optical_frame" +[INFO] [1733933529.557972]: received mask_obs with header seq: 19280 +stamp: + secs: 1733933529 + nsecs: 559420347 +frame_id: '' and grasped_object_mask header seq: 456359 +stamp: + secs: 1733933529 + nsecs: 348522663 +frame_id: "camera_color_optical_frame" +[INFO] [1733933529.766034]: received mask_obs with header seq: 19281 +stamp: + secs: 1733933529 + nsecs: 767741680 +frame_id: '' and grasped_object_mask header seq: 456365 +stamp: + secs: 1733933529 + nsecs: 548133850 +frame_id: "camera_color_optical_frame" +[INFO] [1733933529.965659]: received mask_obs with header seq: 19282 +stamp: + secs: 1733933529 + nsecs: 967270135 +frame_id: '' and grasped_object_mask header seq: 456371 +stamp: + secs: 1733933529 + nsecs: 747733116 +frame_id: "camera_color_optical_frame" +[INFO] [1733933530.177364]: received mask_obs with header seq: 19283 +stamp: + secs: 1733933530 + nsecs: 179042339 +frame_id: '' and grasped_object_mask header seq: 456377 +stamp: + secs: 1733933529 + nsecs: 947320700 +frame_id: "camera_color_optical_frame" +[INFO] [1733933530.387915]: received mask_obs with header seq: 19284 +stamp: + secs: 1733933530 + nsecs: 373801469 +frame_id: '' and grasped_object_mask header seq: 456383 +stamp: + secs: 1733933530 + nsecs: 146897078 +frame_id: "camera_color_optical_frame" +[INFO] [1733933530.584646]: received mask_obs with header seq: 19285 +stamp: + secs: 1733933530 + nsecs: 586524724 +frame_id: '' and grasped_object_mask header seq: 456389 +stamp: + secs: 1733933530 + nsecs: 346498728 +frame_id: "camera_color_optical_frame" +[INFO] [1733933530.778947]: received mask_obs with header seq: 19286 +stamp: + secs: 1733933530 + nsecs: 780860900 +frame_id: '' and grasped_object_mask header seq: 456395 +stamp: + secs: 1733933530 + nsecs: 546112537 +frame_id: "camera_color_optical_frame" +[INFO] [1733933530.975128]: received mask_obs with header seq: 19287 +stamp: + secs: 1733933530 + nsecs: 976986885 +frame_id: '' and grasped_object_mask header seq: 456401 +stamp: + secs: 1733933530 + nsecs: 745714664 +frame_id: "camera_color_optical_frame" +[INFO] [1733933531.176707]: received mask_obs with header seq: 19288 +stamp: + secs: 1733933531 + nsecs: 177587985 +frame_id: '' and grasped_object_mask header seq: 456407 +stamp: + secs: 1733933530 + nsecs: 945305347 +frame_id: "camera_color_optical_frame" +[INFO] [1733933531.375962]: received mask_obs with header seq: 19289 +stamp: + secs: 1733933531 + nsecs: 377348661 +frame_id: '' and grasped_object_mask header seq: 456413 +stamp: + secs: 1733933531 + nsecs: 144884825 +frame_id: "camera_color_optical_frame" +[INFO] [1733933531.576417]: received mask_obs with header seq: 19290 +stamp: + secs: 1733933531 + nsecs: 577230453 +frame_id: '' and grasped_object_mask header seq: 456419 +stamp: + secs: 1733933531 + nsecs: 344480515 +frame_id: "camera_color_optical_frame" +[INFO] [1733933531.774381]: received mask_obs with header seq: 19291 +stamp: + secs: 1733933531 + nsecs: 774772167 +frame_id: '' and grasped_object_mask header seq: 456425 +stamp: + secs: 1733933531 + nsecs: 544083595 +frame_id: "camera_color_optical_frame" +[INFO] [1733933531.974241]: received mask_obs with header seq: 19292 +stamp: + secs: 1733933531 + nsecs: 974217176 +frame_id: '' and grasped_object_mask header seq: 456431 +stamp: + secs: 1733933531 + nsecs: 743675470 +frame_id: "camera_color_optical_frame" +[INFO] [1733933532.181935]: received mask_obs with header seq: 19293 +stamp: + secs: 1733933532 + nsecs: 182882547 +frame_id: '' and grasped_object_mask header seq: 456437 +stamp: + secs: 1733933531 + nsecs: 943254709 +frame_id: "camera_color_optical_frame" +[INFO] [1733933532.376929]: received mask_obs with header seq: 19294 +stamp: + secs: 1733933532 + nsecs: 378683805 +frame_id: '' and grasped_object_mask header seq: 456443 +stamp: + secs: 1733933532 + nsecs: 142821789 +frame_id: "camera_color_optical_frame" +[INFO] [1733933532.573580]: received mask_obs with header seq: 19295 +stamp: + secs: 1733933532 + nsecs: 575046539 +frame_id: '' and grasped_object_mask header seq: 456449 +stamp: + secs: 1733933532 + nsecs: 342411995 +frame_id: "camera_color_optical_frame" +[INFO] [1733933532.778688]: received mask_obs with header seq: 19296 +stamp: + secs: 1733933532 + nsecs: 780057430 +frame_id: '' and grasped_object_mask header seq: 456455 +stamp: + secs: 1733933532 + nsecs: 542014599 +frame_id: "camera_color_optical_frame" +[INFO] [1733933532.981729]: received mask_obs with header seq: 19297 +stamp: + secs: 1733933532 + nsecs: 972363710 +frame_id: '' and grasped_object_mask header seq: 456461 +stamp: + secs: 1733933532 + nsecs: 741605520 +frame_id: "camera_color_optical_frame" +[INFO] [1733933533.178029]: received mask_obs with header seq: 19298 +stamp: + secs: 1733933533 + nsecs: 179881572 +frame_id: '' and grasped_object_mask header seq: 456467 +stamp: + secs: 1733933532 + nsecs: 941184998 +frame_id: "camera_color_optical_frame" +[INFO] [1733933533.373550]: received mask_obs with header seq: 19299 +stamp: + secs: 1733933533 + nsecs: 374586343 +frame_id: '' and grasped_object_mask header seq: 456473 +stamp: + secs: 1733933533 + nsecs: 140751600 +frame_id: "camera_color_optical_frame" +[INFO] [1733933533.572969]: received mask_obs with header seq: 19300 +stamp: + secs: 1733933533 + nsecs: 573492765 +frame_id: '' and grasped_object_mask header seq: 456479 +stamp: + secs: 1733933533 + nsecs: 340344667 +frame_id: "camera_color_optical_frame" +[INFO] [1733933533.774816]: received mask_obs with header seq: 19301 +stamp: + secs: 1733933533 + nsecs: 776640415 +frame_id: '' and grasped_object_mask header seq: 456485 +stamp: + secs: 1733933533 + nsecs: 539952278 +frame_id: "camera_color_optical_frame" +[INFO] [1733933533.982234]: received mask_obs with header seq: 19302 +stamp: + secs: 1733933533 + nsecs: 973614931 +frame_id: '' and grasped_object_mask header seq: 456491 +stamp: + secs: 1733933533 + nsecs: 739548206 +frame_id: "camera_color_optical_frame" +[INFO] [1733933534.187356]: received mask_obs with header seq: 19303 +stamp: + secs: 1733933534 + nsecs: 189149618 +frame_id: '' and grasped_object_mask header seq: 456497 +stamp: + secs: 1733933533 + nsecs: 939132214 +frame_id: "camera_color_optical_frame" +[INFO] [1733933534.381532]: received mask_obs with header seq: 19304 +stamp: + secs: 1733933534 + nsecs: 383198976 +frame_id: '' and grasped_object_mask header seq: 456503 +stamp: + secs: 1733933534 + nsecs: 138705254 +frame_id: "camera_color_optical_frame" +[INFO] [1733933534.576739]: received mask_obs with header seq: 19305 +stamp: + secs: 1733933534 + nsecs: 578289270 +frame_id: '' and grasped_object_mask header seq: 456509 +stamp: + secs: 1733933534 + nsecs: 339050293 +frame_id: "camera_color_optical_frame" +[INFO] [1733933534.777364]: received mask_obs with header seq: 19306 +stamp: + secs: 1733933534 + nsecs: 779097557 +frame_id: '' and grasped_object_mask header seq: 456515 +stamp: + secs: 1733933534 + nsecs: 540019274 +frame_id: "camera_color_optical_frame" +[INFO] [1733933534.977561]: received mask_obs with header seq: 19307 +stamp: + secs: 1733933534 + nsecs: 979324579 +frame_id: '' and grasped_object_mask header seq: 456520 +stamp: + secs: 1733933534 + nsecs: 707561970 +frame_id: "camera_color_optical_frame" +[INFO] [1733933535.180796]: received mask_obs with header seq: 19308 +stamp: + secs: 1733933535 + nsecs: 182394981 +frame_id: '' and grasped_object_mask header seq: 456520 +stamp: + secs: 1733933534 + nsecs: 707561970 +frame_id: "camera_color_optical_frame" +[INFO] [1733933535.365322]: received mask_obs with header seq: 19309 +stamp: + secs: 1733933535 + nsecs: 366986513 +frame_id: '' and grasped_object_mask header seq: 456533 +stamp: + secs: 1733933535 + nsecs: 143459558 +frame_id: "camera_color_optical_frame" +[INFO] [1733933535.566100]: received mask_obs with header seq: 19310 +stamp: + secs: 1733933535 + nsecs: 567796468 +frame_id: '' and grasped_object_mask header seq: 456539 +stamp: + secs: 1733933535 + nsecs: 343913078 +frame_id: "camera_color_optical_frame" +[INFO] [1733933535.777553]: received mask_obs with header seq: 19311 +stamp: + secs: 1733933535 + nsecs: 779175996 +frame_id: '' and grasped_object_mask header seq: 456545 +stamp: + secs: 1733933535 + nsecs: 543759823 +frame_id: "camera_color_optical_frame" +[INFO] [1733933535.971772]: received mask_obs with header seq: 19312 +stamp: + secs: 1733933535 + nsecs: 973637819 +frame_id: '' and grasped_object_mask header seq: 456551 +stamp: + secs: 1733933535 + nsecs: 743607044 +frame_id: "camera_color_optical_frame" +[INFO] [1733933536.174431]: received mask_obs with header seq: 19313 +stamp: + secs: 1733933536 + nsecs: 176326990 +frame_id: '' and grasped_object_mask header seq: 456557 +stamp: + secs: 1733933535 + nsecs: 943454266 +frame_id: "camera_color_optical_frame" +[INFO] [1733933536.364143]: received mask_obs with header seq: 19314 +stamp: + secs: 1733933536 + nsecs: 366081953 +frame_id: '' and grasped_object_mask header seq: 456563 +stamp: + secs: 1733933536 + nsecs: 143302441 +frame_id: "camera_color_optical_frame" +[INFO] [1733933536.566257]: received mask_obs with header seq: 19315 +stamp: + secs: 1733933536 + nsecs: 567502260 +frame_id: '' and grasped_object_mask header seq: 456569 +stamp: + secs: 1733933536 + nsecs: 343147516 +frame_id: "camera_color_optical_frame" +[INFO] [1733933536.768507]: received mask_obs with header seq: 19316 +stamp: + secs: 1733933536 + nsecs: 769946098 +frame_id: '' and grasped_object_mask header seq: 456575 +stamp: + secs: 1733933536 + nsecs: 542990923 +frame_id: "camera_color_optical_frame" +[INFO] [1733933536.965400]: received mask_obs with header seq: 19317 +stamp: + secs: 1733933536 + nsecs: 967284202 +frame_id: '' and grasped_object_mask header seq: 456581 +stamp: + secs: 1733933536 + nsecs: 742833614 +frame_id: "camera_color_optical_frame" +[INFO] [1733933537.165715]: received mask_obs with header seq: 19318 +stamp: + secs: 1733933537 + nsecs: 167181730 +frame_id: '' and grasped_object_mask header seq: 456587 +stamp: + secs: 1733933536 + nsecs: 942676306 +frame_id: "camera_color_optical_frame" +[INFO] [1733933537.365900]: received mask_obs with header seq: 19319 +stamp: + secs: 1733933537 + nsecs: 367402315 +frame_id: '' and grasped_object_mask header seq: 456593 +stamp: + secs: 1733933537 + nsecs: 142518759 +frame_id: "camera_color_optical_frame" +[INFO] [1733933537.566750]: received mask_obs with header seq: 19320 +stamp: + secs: 1733933537 + nsecs: 568576574 +frame_id: '' and grasped_object_mask header seq: 456599 +stamp: + secs: 1733933537 + nsecs: 342359066 +frame_id: "camera_color_optical_frame" +[INFO] [1733933537.767088]: received mask_obs with header seq: 19321 +stamp: + secs: 1733933537 + nsecs: 768725156 +frame_id: '' and grasped_object_mask header seq: 456605 +stamp: + secs: 1733933537 + nsecs: 542197227 +frame_id: "camera_color_optical_frame" +[INFO] [1733933537.974159]: received mask_obs with header seq: 19322 +stamp: + secs: 1733933537 + nsecs: 975896596 +frame_id: '' and grasped_object_mask header seq: 456611 +stamp: + secs: 1733933537 + nsecs: 742036581 +frame_id: "camera_color_optical_frame" +[INFO] [1733933538.166317]: received mask_obs with header seq: 19323 +stamp: + secs: 1733933538 + nsecs: 167468070 +frame_id: '' and grasped_object_mask header seq: 456617 +stamp: + secs: 1733933537 + nsecs: 941874504 +frame_id: "camera_color_optical_frame" +[INFO] [1733933538.366187]: received mask_obs with header seq: 19324 +stamp: + secs: 1733933538 + nsecs: 367975950 +frame_id: '' and grasped_object_mask header seq: 456623 +stamp: + secs: 1733933538 + nsecs: 141712427 +frame_id: "camera_color_optical_frame" +[INFO] [1733933538.566085]: received mask_obs with header seq: 19325 +stamp: + secs: 1733933538 + nsecs: 567611455 +frame_id: '' and grasped_object_mask header seq: 456629 +stamp: + secs: 1733933538 + nsecs: 341554403 +frame_id: "camera_color_optical_frame" +[INFO] [1733933538.768729]: received mask_obs with header seq: 19326 +stamp: + secs: 1733933538 + nsecs: 770436048 +frame_id: '' and grasped_object_mask header seq: 456635 +stamp: + secs: 1733933538 + nsecs: 541399479 +frame_id: "camera_color_optical_frame" +[INFO] [1733933538.974284]: received mask_obs with header seq: 19327 +stamp: + secs: 1733933538 + nsecs: 976132154 +frame_id: '' and grasped_object_mask header seq: 456641 +stamp: + secs: 1733933538 + nsecs: 741244316 +frame_id: "camera_color_optical_frame" +[INFO] [1733933539.168756]: received mask_obs with header seq: 19328 +stamp: + secs: 1733933539 + nsecs: 170695066 +frame_id: '' and grasped_object_mask header seq: 456647 +stamp: + secs: 1733933538 + nsecs: 941089869 +frame_id: "camera_color_optical_frame" +[INFO] [1733933539.366569]: received mask_obs with header seq: 19329 +stamp: + secs: 1733933539 + nsecs: 368016719 +frame_id: '' and grasped_object_mask header seq: 456653 +stamp: + secs: 1733933539 + nsecs: 140936375 +frame_id: "camera_color_optical_frame" +[INFO] [1733933539.566686]: received mask_obs with header seq: 19330 +stamp: + secs: 1733933539 + nsecs: 568303823 +frame_id: '' and grasped_object_mask header seq: 456659 +stamp: + secs: 1733933539 + nsecs: 340775251 +frame_id: "camera_color_optical_frame" +[INFO] [1733933539.768176]: received mask_obs with header seq: 19331 +stamp: + secs: 1733933539 + nsecs: 769976377 +frame_id: '' and grasped_object_mask header seq: 456665 +stamp: + secs: 1733933539 + nsecs: 540608644 +frame_id: "camera_color_optical_frame" +[INFO] [1733933539.967711]: received mask_obs with header seq: 19332 +stamp: + secs: 1733933539 + nsecs: 968876123 +frame_id: '' and grasped_object_mask header seq: 456671 +stamp: + secs: 1733933539 + nsecs: 740441561 +frame_id: "camera_color_optical_frame" +[INFO] [1733933540.168060]: received mask_obs with header seq: 19333 +stamp: + secs: 1733933540 + nsecs: 169571876 +frame_id: '' and grasped_object_mask header seq: 456677 +stamp: + secs: 1733933539 + nsecs: 940274715 +frame_id: "camera_color_optical_frame" +[INFO] [1733933540.368221]: received mask_obs with header seq: 19334 +stamp: + secs: 1733933540 + nsecs: 370055437 +frame_id: '' and grasped_object_mask header seq: 456683 +stamp: + secs: 1733933540 + nsecs: 140107155 +frame_id: "camera_color_optical_frame" +[INFO] [1733933540.569060]: received mask_obs with header seq: 19335 +stamp: + secs: 1733933540 + nsecs: 570863008 +frame_id: '' and grasped_object_mask header seq: 456689 +stamp: + secs: 1733933540 + nsecs: 339942455 +frame_id: "camera_color_optical_frame" +[INFO] [1733933540.770929]: received mask_obs with header seq: 19336 +stamp: + secs: 1733933540 + nsecs: 772789239 +frame_id: '' and grasped_object_mask header seq: 456695 +stamp: + secs: 1733933540 + nsecs: 539777756 +frame_id: "camera_color_optical_frame" +[INFO] [1733933540.970255]: received mask_obs with header seq: 19337 +stamp: + secs: 1733933540 + nsecs: 972033977 +frame_id: '' and grasped_object_mask header seq: 456701 +stamp: + secs: 1733933540 + nsecs: 739612818 +frame_id: "camera_color_optical_frame" +[INFO] [1733933541.168107]: received mask_obs with header seq: 19338 +stamp: + secs: 1733933541 + nsecs: 169883251 +frame_id: '' and grasped_object_mask header seq: 456707 +stamp: + secs: 1733933540 + nsecs: 939447641 +frame_id: "camera_color_optical_frame" +[INFO] [1733933541.368499]: received mask_obs with header seq: 19339 +stamp: + secs: 1733933541 + nsecs: 370128393 +frame_id: '' and grasped_object_mask header seq: 456713 +stamp: + secs: 1733933541 + nsecs: 139282227 +frame_id: "camera_color_optical_frame" +[INFO] [1733933541.567134]: received mask_obs with header seq: 19340 +stamp: + secs: 1733933541 + nsecs: 569024801 +frame_id: '' and grasped_object_mask header seq: 456719 +stamp: + secs: 1733933541 + nsecs: 339122534 +frame_id: "camera_color_optical_frame" +[INFO] [1733933541.781714]: received mask_obs with header seq: 19341 +stamp: + secs: 1733933541 + nsecs: 783623218 +frame_id: '' and grasped_object_mask header seq: 456725 +stamp: + secs: 1733933541 + nsecs: 538968325 +frame_id: "camera_color_optical_frame" +[INFO] [1733933541.974065]: received mask_obs with header seq: 19342 +stamp: + secs: 1733933541 + nsecs: 975905418 +frame_id: '' and grasped_object_mask header seq: 456731 +stamp: + secs: 1733933541 + nsecs: 738813162 +frame_id: "camera_color_optical_frame" +[INFO] [1733933542.171940]: received mask_obs with header seq: 19343 +stamp: + secs: 1733933542 + nsecs: 173725128 +frame_id: '' and grasped_object_mask header seq: 456737 +stamp: + secs: 1733933541 + nsecs: 938658237 +frame_id: "camera_color_optical_frame" +[INFO] [1733933542.369936]: received mask_obs with header seq: 19344 +stamp: + secs: 1733933542 + nsecs: 371735095 +frame_id: '' and grasped_object_mask header seq: 456743 +stamp: + secs: 1733933542 + nsecs: 138503313 +frame_id: "camera_color_optical_frame" +[INFO] [1733933542.567875]: received mask_obs with header seq: 19345 +stamp: + secs: 1733933542 + nsecs: 569837331 +frame_id: '' and grasped_object_mask header seq: 456749 +stamp: + secs: 1733933542 + nsecs: 338371754 +frame_id: "camera_color_optical_frame" +[INFO] [1733933542.769596]: received mask_obs with header seq: 19346 +stamp: + secs: 1733933542 + nsecs: 771248340 +frame_id: '' and grasped_object_mask header seq: 456755 +stamp: + secs: 1733933542 + nsecs: 538259268 +frame_id: "camera_color_optical_frame" +[INFO] [1733933542.966531]: received mask_obs with header seq: 19347 +stamp: + secs: 1733933542 + nsecs: 967945098 +frame_id: '' and grasped_object_mask header seq: 456761 +stamp: + secs: 1733933542 + nsecs: 738149405 +frame_id: "camera_color_optical_frame" +[INFO] [1733933543.167890]: received mask_obs with header seq: 19348 +stamp: + secs: 1733933543 + nsecs: 169655323 +frame_id: '' and grasped_object_mask header seq: 456767 +stamp: + secs: 1733933542 + nsecs: 938040257 +frame_id: "camera_color_optical_frame" +[INFO] [1733933543.368555]: received mask_obs with header seq: 19349 +stamp: + secs: 1733933543 + nsecs: 370202779 +frame_id: '' and grasped_object_mask header seq: 456773 +stamp: + secs: 1733933543 + nsecs: 137932777 +frame_id: "camera_color_optical_frame" +[INFO] [1733933543.566443]: received mask_obs with header seq: 19350 +stamp: + secs: 1733933543 + nsecs: 568337917 +frame_id: '' and grasped_object_mask header seq: 456779 +stamp: + secs: 1733933543 + nsecs: 337794781 +frame_id: "camera_color_optical_frame" +[INFO] [1733933543.768569]: received mask_obs with header seq: 19351 +stamp: + secs: 1733933543 + nsecs: 770536661 +frame_id: '' and grasped_object_mask header seq: 456785 +stamp: + secs: 1733933543 + nsecs: 537627697 +frame_id: "camera_color_optical_frame" +[INFO] [1733933543.966228]: received mask_obs with header seq: 19352 +stamp: + secs: 1733933543 + nsecs: 968189477 +frame_id: '' and grasped_object_mask header seq: 456791 +stamp: + secs: 1733933543 + nsecs: 737460375 +frame_id: "camera_color_optical_frame" +[INFO] [1733933544.167936]: received mask_obs with header seq: 19353 +stamp: + secs: 1733933544 + nsecs: 169728994 +frame_id: '' and grasped_object_mask header seq: 456797 +stamp: + secs: 1733933543 + nsecs: 937293530 +frame_id: "camera_color_optical_frame" +[INFO] [1733933544.366201]: received mask_obs with header seq: 19354 +stamp: + secs: 1733933544 + nsecs: 367969989 +frame_id: '' and grasped_object_mask header seq: 456803 +stamp: + secs: 1733933544 + nsecs: 137125254 +frame_id: "camera_color_optical_frame" +[INFO] [1733933544.575135]: received mask_obs with header seq: 19355 +stamp: + secs: 1733933544 + nsecs: 576985836 +frame_id: '' and grasped_object_mask header seq: 456809 +stamp: + secs: 1733933544 + nsecs: 336959124 +frame_id: "camera_color_optical_frame" +[INFO] [1733933544.772749]: received mask_obs with header seq: 19356 +stamp: + secs: 1733933544 + nsecs: 774513721 +frame_id: '' and grasped_object_mask header seq: 456815 +stamp: + secs: 1733933544 + nsecs: 536795616 +frame_id: "camera_color_optical_frame" +[INFO] [1733933544.965987]: received mask_obs with header seq: 19357 +stamp: + secs: 1733933544 + nsecs: 967933654 +frame_id: '' and grasped_object_mask header seq: 456821 +stamp: + secs: 1733933544 + nsecs: 736631870 +frame_id: "camera_color_optical_frame" +[INFO] [1733933545.166615]: received mask_obs with header seq: 19358 +stamp: + secs: 1733933545 + nsecs: 168164968 +frame_id: '' and grasped_object_mask header seq: 456827 +stamp: + secs: 1733933544 + nsecs: 936467886 +frame_id: "camera_color_optical_frame" +[INFO] [1733933545.366173]: received mask_obs with header seq: 19359 +stamp: + secs: 1733933545 + nsecs: 367934703 +frame_id: '' and grasped_object_mask header seq: 456833 +stamp: + secs: 1733933545 + nsecs: 136303663 +frame_id: "camera_color_optical_frame" +[INFO] [1733933545.566532]: received mask_obs with header seq: 19360 +stamp: + secs: 1733933545 + nsecs: 568428039 +frame_id: '' and grasped_object_mask header seq: 456839 +stamp: + secs: 1733933545 + nsecs: 336140394 +frame_id: "camera_color_optical_frame" +[INFO] [1733933545.766991]: received mask_obs with header seq: 19361 +stamp: + secs: 1733933545 + nsecs: 768800497 +frame_id: '' and grasped_object_mask header seq: 456845 +stamp: + secs: 1733933545 + nsecs: 535975695 +frame_id: "camera_color_optical_frame" +[INFO] [1733933545.966151]: received mask_obs with header seq: 19362 +stamp: + secs: 1733933545 + nsecs: 968106269 +frame_id: '' and grasped_object_mask header seq: 456851 +stamp: + secs: 1733933545 + nsecs: 735810518 +frame_id: "camera_color_optical_frame" +[INFO] [1733933546.170038]: received mask_obs with header seq: 19363 +stamp: + secs: 1733933546 + nsecs: 171672344 +frame_id: '' and grasped_object_mask header seq: 456857 +stamp: + secs: 1733933545 + nsecs: 935645342 +frame_id: "camera_color_optical_frame" +[INFO] [1733933546.367046]: received mask_obs with header seq: 19364 +stamp: + secs: 1733933546 + nsecs: 368568897 +frame_id: '' and grasped_object_mask header seq: 456863 +stamp: + secs: 1733933546 + nsecs: 135479689 +frame_id: "camera_color_optical_frame" +[INFO] [1733933546.584535]: received mask_obs with header seq: 19365 +stamp: + secs: 1733933546 + nsecs: 584922313 +frame_id: '' and grasped_object_mask header seq: 456869 +stamp: + secs: 1733933546 + nsecs: 335820198 +frame_id: "camera_color_optical_frame" +[INFO] [1733933546.784385]: received mask_obs with header seq: 19366 +stamp: + secs: 1733933546 + nsecs: 786313056 +frame_id: '' and grasped_object_mask header seq: 456876 +stamp: + secs: 1733933546 + nsecs: 571707726 +frame_id: "camera_color_optical_frame" +[INFO] [1733933546.971997]: received mask_obs with header seq: 19367 +stamp: + secs: 1733933546 + nsecs: 973802089 +frame_id: '' and grasped_object_mask header seq: 456881 +stamp: + secs: 1733933546 + nsecs: 740272284 +frame_id: "camera_color_optical_frame" +[INFO] [1733933547.167536]: received mask_obs with header seq: 19368 +stamp: + secs: 1733933547 + nsecs: 169034719 +frame_id: '' and grasped_object_mask header seq: 456887 +stamp: + secs: 1733933546 + nsecs: 942629814 +frame_id: "camera_color_optical_frame" +[INFO] [1733933547.365142]: received mask_obs with header seq: 19369 +stamp: + secs: 1733933547 + nsecs: 366913318 +frame_id: '' and grasped_object_mask header seq: 456893 +stamp: + secs: 1733933547 + nsecs: 145075083 +frame_id: "camera_color_optical_frame" +[INFO] [1733933547.565371]: received mask_obs with header seq: 19370 +stamp: + secs: 1733933547 + nsecs: 566951274 +frame_id: '' and grasped_object_mask header seq: 456899 +stamp: + secs: 1733933547 + nsecs: 346969366 +frame_id: "camera_color_optical_frame" +[INFO] [1733933547.766907]: received mask_obs with header seq: 19371 +stamp: + secs: 1733933547 + nsecs: 768724679 +frame_id: '' and grasped_object_mask header seq: 456905 +stamp: + secs: 1733933547 + nsecs: 546586752 +frame_id: "camera_color_optical_frame" +[INFO] [1733933547.968614]: received mask_obs with header seq: 19372 +stamp: + secs: 1733933547 + nsecs: 970469951 +frame_id: '' and grasped_object_mask header seq: 456911 +stamp: + secs: 1733933547 + nsecs: 746193171 +frame_id: "camera_color_optical_frame" +[INFO] [1733933548.167430]: received mask_obs with header seq: 19373 +stamp: + secs: 1733933548 + nsecs: 169167518 +frame_id: '' and grasped_object_mask header seq: 456917 +stamp: + secs: 1733933547 + nsecs: 945786953 +frame_id: "camera_color_optical_frame" +[INFO] [1733933548.367364]: received mask_obs with header seq: 19374 +stamp: + secs: 1733933548 + nsecs: 368965864 +frame_id: '' and grasped_object_mask header seq: 456923 +stamp: + secs: 1733933548 + nsecs: 145369053 +frame_id: "camera_color_optical_frame" +[INFO] [1733933548.565527]: received mask_obs with header seq: 19375 +stamp: + secs: 1733933548 + nsecs: 567505836 +frame_id: '' and grasped_object_mask header seq: 456929 +stamp: + secs: 1733933548 + nsecs: 344947815 +frame_id: "camera_color_optical_frame" +[INFO] [1733933548.767768]: received mask_obs with header seq: 19376 +stamp: + secs: 1733933548 + nsecs: 769680500 +frame_id: '' and grasped_object_mask header seq: 456935 +stamp: + secs: 1733933548 + nsecs: 544544220 +frame_id: "camera_color_optical_frame" +[INFO] [1733933548.965278]: received mask_obs with header seq: 19377 +stamp: + secs: 1733933548 + nsecs: 967165231 +frame_id: '' and grasped_object_mask header seq: 456941 +stamp: + secs: 1733933548 + nsecs: 744128466 +frame_id: "camera_color_optical_frame" +[INFO] [1733933549.168469]: received mask_obs with header seq: 19378 +stamp: + secs: 1733933549 + nsecs: 170257806 +frame_id: '' and grasped_object_mask header seq: 456947 +stamp: + secs: 1733933548 + nsecs: 943701029 +frame_id: "camera_color_optical_frame" +[INFO] [1733933549.367894]: received mask_obs with header seq: 19379 +stamp: + secs: 1733933549 + nsecs: 369774341 +frame_id: '' and grasped_object_mask header seq: 456953 +stamp: + secs: 1733933549 + nsecs: 143260241 +frame_id: "camera_color_optical_frame" +[INFO] [1733933549.565538]: received mask_obs with header seq: 19380 +stamp: + secs: 1733933549 + nsecs: 567011833 +frame_id: '' and grasped_object_mask header seq: 456959 +stamp: + secs: 1733933549 + nsecs: 342819452 +frame_id: "camera_color_optical_frame" +[INFO] [1733933549.782760]: received mask_obs with header seq: 19381 +stamp: + secs: 1733933549 + nsecs: 784362554 +frame_id: '' and grasped_object_mask header seq: 456965 +stamp: + secs: 1733933549 + nsecs: 542412758 +frame_id: "camera_color_optical_frame" +[INFO] [1733933549.988972]: received mask_obs with header seq: 19382 +stamp: + secs: 1733933549 + nsecs: 990698814 +frame_id: '' and grasped_object_mask header seq: 456972 +stamp: + secs: 1733933549 + nsecs: 775256634 +frame_id: "camera_color_optical_frame" +[INFO] [1733933550.189059]: received mask_obs with header seq: 19383 +stamp: + secs: 1733933550 + nsecs: 190525531 +frame_id: '' and grasped_object_mask header seq: 456977 +stamp: + secs: 1733933549 + nsecs: 941562176 +frame_id: "camera_color_optical_frame" +[INFO] [1733933550.393912]: received mask_obs with header seq: 19384 +stamp: + secs: 1733933550 + nsecs: 395380258 +frame_id: '' and grasped_object_mask header seq: 456984 +stamp: + secs: 1733933550 + nsecs: 174376965 +frame_id: "camera_color_optical_frame" +[INFO] [1733933550.585112]: received mask_obs with header seq: 19385 +stamp: + secs: 1733933550 + nsecs: 587037086 +frame_id: '' and grasped_object_mask header seq: 456990 +stamp: + secs: 1733933550 + nsecs: 373944044 +frame_id: "camera_color_optical_frame" +[INFO] [1733933550.785770]: received mask_obs with header seq: 19386 +stamp: + secs: 1733933550 + nsecs: 787494182 +frame_id: '' and grasped_object_mask header seq: 456996 +stamp: + secs: 1733933550 + nsecs: 573541164 +frame_id: "camera_color_optical_frame" +[INFO] [1733933550.980799]: received mask_obs with header seq: 19387 +stamp: + secs: 1733933550 + nsecs: 982715129 +frame_id: '' and grasped_object_mask header seq: 457002 +stamp: + secs: 1733933550 + nsecs: 773127079 +frame_id: "camera_color_optical_frame" +[INFO] [1733933551.181290]: received mask_obs with header seq: 19388 +stamp: + secs: 1733933551 + nsecs: 180248737 +frame_id: '' and grasped_object_mask header seq: 457008 +stamp: + secs: 1733933550 + nsecs: 972699881 +frame_id: "camera_color_optical_frame" +[INFO] [1733933551.370754]: received mask_obs with header seq: 19389 +stamp: + secs: 1733933551 + nsecs: 372632741 +frame_id: '' and grasped_object_mask header seq: 457014 +stamp: + secs: 1733933551 + nsecs: 172260523 +frame_id: "camera_color_optical_frame" +[INFO] [1733933551.569416]: received mask_obs with header seq: 19390 +stamp: + secs: 1733933551 + nsecs: 571345567 +frame_id: '' and grasped_object_mask header seq: 457019 +stamp: + secs: 1733933551 + nsecs: 338563442 +frame_id: "camera_color_optical_frame" +[INFO] [1733933551.770987]: received mask_obs with header seq: 19391 +stamp: + secs: 1733933551 + nsecs: 772728204 +frame_id: '' and grasped_object_mask header seq: 457025 +stamp: + secs: 1733933551 + nsecs: 538157225 +frame_id: "camera_color_optical_frame" +[INFO] [1733933551.968793]: received mask_obs with header seq: 19392 +stamp: + secs: 1733933551 + nsecs: 970456838 +frame_id: '' and grasped_object_mask header seq: 457031 +stamp: + secs: 1733933551 + nsecs: 737738609 +frame_id: "camera_color_optical_frame" +[INFO] [1733933552.168632]: received mask_obs with header seq: 19393 +stamp: + secs: 1733933552 + nsecs: 170408725 +frame_id: '' and grasped_object_mask header seq: 457037 +stamp: + secs: 1733933551 + nsecs: 937307358 +frame_id: "camera_color_optical_frame" +[INFO] [1733933552.367222]: received mask_obs with header seq: 19394 +stamp: + secs: 1733933552 + nsecs: 368818521 +frame_id: '' and grasped_object_mask header seq: 457043 +stamp: + secs: 1733933552 + nsecs: 136863708 +frame_id: "camera_color_optical_frame" +[INFO] [1733933552.567692]: received mask_obs with header seq: 19395 +stamp: + secs: 1733933552 + nsecs: 569620370 +frame_id: '' and grasped_object_mask header seq: 457049 +stamp: + secs: 1733933552 + nsecs: 336422920 +frame_id: "camera_color_optical_frame" +[INFO] [1733933552.769928]: received mask_obs with header seq: 19396 +stamp: + secs: 1733933552 + nsecs: 771802902 +frame_id: '' and grasped_object_mask header seq: 457055 +stamp: + secs: 1733933552 + nsecs: 536029339 +frame_id: "camera_color_optical_frame" +[INFO] [1733933552.971576]: received mask_obs with header seq: 19397 +stamp: + secs: 1733933552 + nsecs: 973117113 +frame_id: '' and grasped_object_mask header seq: 457061 +stamp: + secs: 1733933552 + nsecs: 735624790 +frame_id: "camera_color_optical_frame" +[INFO] [1733933553.169343]: received mask_obs with header seq: 19398 +stamp: + secs: 1733933553 + nsecs: 170985698 +frame_id: '' and grasped_object_mask header seq: 457067 +stamp: + secs: 1733933552 + nsecs: 935207367 +frame_id: "camera_color_optical_frame" +[INFO] [1733933553.385072]: received mask_obs with header seq: 19399 +stamp: + secs: 1733933553 + nsecs: 386411428 +frame_id: '' and grasped_object_mask header seq: 457073 +stamp: + secs: 1733933553 + nsecs: 134778023 +frame_id: "camera_color_optical_frame" +[INFO] [1733933553.584490]: received mask_obs with header seq: 19400 +stamp: + secs: 1733933553 + nsecs: 585329294 +frame_id: '' and grasped_object_mask header seq: 457080 +stamp: + secs: 1733933553 + nsecs: 367617607 +frame_id: "camera_color_optical_frame" +[INFO] [1733933553.782771]: received mask_obs with header seq: 19401 +stamp: + secs: 1733933553 + nsecs: 783700942 +frame_id: '' and grasped_object_mask header seq: 457086 +stamp: + secs: 1733933553 + nsecs: 567221403 +frame_id: "camera_color_optical_frame" +[INFO] [1733933553.977867]: received mask_obs with header seq: 19402 +stamp: + secs: 1733933553 + nsecs: 979053974 +frame_id: '' and grasped_object_mask header seq: 457092 +stamp: + secs: 1733933553 + nsecs: 766813278 +frame_id: "camera_color_optical_frame" +[INFO] [1733933554.176917]: received mask_obs with header seq: 19403 +stamp: + secs: 1733933554 + nsecs: 177805662 +frame_id: '' and grasped_object_mask header seq: 457097 +stamp: + secs: 1733933553 + nsecs: 933131218 +frame_id: "camera_color_optical_frame" +[INFO] [1733933554.371681]: received mask_obs with header seq: 19404 +stamp: + secs: 1733933554 + nsecs: 373611450 +frame_id: '' and grasped_object_mask header seq: 457103 +stamp: + secs: 1733933554 + nsecs: 132700920 +frame_id: "camera_color_optical_frame" +[INFO] [1733933554.572587]: received mask_obs with header seq: 19405 +stamp: + secs: 1733933554 + nsecs: 574538230 +frame_id: '' and grasped_object_mask header seq: 457109 +stamp: + secs: 1733933554 + nsecs: 332272053 +frame_id: "camera_color_optical_frame" +[INFO] [1733933554.772655]: received mask_obs with header seq: 19406 +stamp: + secs: 1733933554 + nsecs: 774507045 +frame_id: '' and grasped_object_mask header seq: 457115 +stamp: + secs: 1733933554 + nsecs: 531884670 +frame_id: "camera_color_optical_frame" +[INFO] [1733933554.977954]: received mask_obs with header seq: 19407 +stamp: + secs: 1733933554 + nsecs: 977737426 +frame_id: '' and grasped_object_mask header seq: 457121 +stamp: + secs: 1733933554 + nsecs: 731486082 +frame_id: "camera_color_optical_frame" +[INFO] [1733933555.172410]: received mask_obs with header seq: 19408 +stamp: + secs: 1733933555 + nsecs: 173611879 +frame_id: '' and grasped_object_mask header seq: 457127 +stamp: + secs: 1733933554 + nsecs: 931075811 +frame_id: "camera_color_optical_frame" +[INFO] [1733933555.374397]: received mask_obs with header seq: 19409 +stamp: + secs: 1733933555 + nsecs: 376221656 +frame_id: '' and grasped_object_mask header seq: 457134 +stamp: + secs: 1733933555 + nsecs: 163916588 +frame_id: "camera_color_optical_frame" +[INFO] [1733933555.572475]: received mask_obs with header seq: 19410 +stamp: + secs: 1733933555 + nsecs: 574338674 +frame_id: '' and grasped_object_mask header seq: 457140 +stamp: + secs: 1733933555 + nsecs: 363499641 +frame_id: "camera_color_optical_frame" +[INFO] [1733933555.778793]: received mask_obs with header seq: 19411 +stamp: + secs: 1733933555 + nsecs: 780766963 +frame_id: '' and grasped_object_mask header seq: 457146 +stamp: + secs: 1733933555 + nsecs: 563102484 +frame_id: "camera_color_optical_frame" +[INFO] [1733933555.810666]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1733933555.994950]: received mask_obs with header seq: 19412 +stamp: + secs: 1733933555 + nsecs: 975963592 +frame_id: '' and grasped_object_mask header seq: 457152 +stamp: + secs: 1733933555 + nsecs: 762694120 +frame_id: "camera_color_optical_frame" +[INFO] [1733933556.615831]: Stored episode 5. +[INFO] [1733933556.621963]: Saving video... +[INFO] [1733933556.870713]: Video saved! +[INFO] [1733933556.914403]: resetting environment +[INFO] [1733933556.932846]: cleared current plan +[INFO] [1733933556.938800]: moving to home +[INFO] [1733933559.846259]: reached home +[INFO] [1733933559.853637]: reset action history +[INFO] [1733933562.643130]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! \ No newline at end of file diff --git a/110822/wandb/run-20241211_110828-7k8jf8l5/files/requirements.txt b/110822/wandb/run-20241211_110828-7k8jf8l5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..192179a195956e252bda33f2342b4970a9de79d9 --- /dev/null +++ b/110822/wandb/run-20241211_110828-7k8jf8l5/files/requirements.txt @@ -0,0 +1,339 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.2 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-metadata.json b/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..03ea067f40151ec1250a6a4ba68078dff04e2ece --- /dev/null +++ b/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2024-12-11T16:08:28.905581", + "startedAt": "2024-12-11T16:08:28.497858", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "1b557f164c7f62a8d1545c761b13ede1677c3a83" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.98675, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4499.916, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.021, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.003, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 623.7553443908691 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71596908569336 + } +} diff --git a/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json b/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7c53ee739094dfd6ab45a66a9d7e608ef07e71fa --- /dev/null +++ b/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "a7ce057e6fe1a72a45a4317170a65df82df6555562cdad1d7cd7243949470e1a", "size": 1148214, "path": "media/videos/eval/0_eval_0_a7ce057e6fe1a72a45a4.mp4"}, "global_step": 679, "_timestamp": 1733933565.4088845, "_runtime": 256.90358448028564, "_step": 9, "eval/1_eval": {"_type": "video-file", "sha256": "dce37a039a0f3ae510e5dcc44fee8b8ef5b47c9b606de8657c821538ad6b0900", "size": 1152733, "path": "media/videos/eval/1_eval_1_dce37a039a0f3ae510e5.mp4"}, "eval/num_success": 5.0, "episode": 5.0, "eval/success_rate": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "c7cd658cf3e0e8f2dada2bbb69769df6b96cb1af52105e6b852a5aa6e7dce071", "size": 1195653, "path": "media/videos/eval/2_eval_2_c7cd658cf3e0e8f2dada.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "ac712fc970ab6ee59b1c4c0c66fe4bb4b0ecb93b156e194c41d4bf46045d2870", "size": 1242757, "path": "media/videos/eval/3_eval_3_ac712fc970ab6ee59b1c.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "b9f6613e7cd6916b75191e5752ebdf1ff8273ec0404a1b2cc3197bf4e1dbcc81", "size": 1261098, "path": "media/videos/eval/4_eval_4_b9f6613e7cd6916b7519.mp4"}, "_wandb": {"runtime": 256}} \ No newline at end of file diff --git a/110822/wandb/run-20241211_110828-7k8jf8l5/logs/debug-internal.log b/110822/wandb/run-20241211_110828-7k8jf8l5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..42926a5950f23efcaa809f2543c32bb089dfb298 --- /dev/null +++ b/110822/wandb/run-20241211_110828-7k8jf8l5/logs/debug-internal.log @@ -0,0 +1,758 @@ +2024-12-11 11:08:28,505 INFO StreamThr :736906 [internal.py:wandb_internal():85] W&B internal server running at pid: 736906, started at: 2024-12-11 11:08:28.505002 +2024-12-11 11:08:28,506 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status +2024-12-11 11:08:28,525 INFO WriterThread:736906 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/run-7k8jf8l5.wandb +2024-12-11 11:08:28,525 DEBUG SenderThread:736906 [sender.py:send():379] send: header +2024-12-11 11:08:28,533 DEBUG SenderThread:736906 [sender.py:send():379] send: run +2024-12-11 11:08:28,813 INFO SenderThread:736906 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files +2024-12-11 11:08:28,813 INFO SenderThread:736906 [sender.py:_start_run_threads():1188] run started: 7k8jf8l5 with start time 1733933308.5053 +2024-12-11 11:08:28,832 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: check_version +2024-12-11 11:08:28,833 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: check_version +2024-12-11 11:08:28,887 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: run_start +2024-12-11 11:08:28,891 DEBUG HandlerThread:736906 [system_info.py:__init__():26] System info init +2024-12-11 11:08:28,892 DEBUG HandlerThread:736906 [system_info.py:__init__():41] System info init done +2024-12-11 11:08:28,892 INFO HandlerThread:736906 [system_monitor.py:start():194] Starting system monitor +2024-12-11 11:08:28,892 INFO SystemMonitor:736906 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-11 11:08:28,892 INFO HandlerThread:736906 [system_monitor.py:probe():214] Collecting system info +2024-12-11 11:08:28,893 INFO SystemMonitor:736906 [interfaces.py:start():188] Started cpu monitoring +2024-12-11 11:08:28,893 INFO SystemMonitor:736906 [interfaces.py:start():188] Started disk monitoring +2024-12-11 11:08:28,894 INFO SystemMonitor:736906 [interfaces.py:start():188] Started gpu monitoring +2024-12-11 11:08:28,894 INFO SystemMonitor:736906 [interfaces.py:start():188] Started memory monitoring +2024-12-11 11:08:28,894 INFO SystemMonitor:736906 [interfaces.py:start():188] Started network monitoring +2024-12-11 11:08:28,905 DEBUG HandlerThread:736906 [system_info.py:probe():152] Probing system +2024-12-11 11:08:28,907 DEBUG HandlerThread:736906 [system_info.py:_probe_git():137] Probing git +2024-12-11 11:08:28,913 DEBUG HandlerThread:736906 [system_info.py:_probe_git():145] Probing git done +2024-12-11 11:08:28,913 DEBUG HandlerThread:736906 [system_info.py:probe():200] Probing system done +2024-12-11 11:08:28,913 DEBUG HandlerThread:736906 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-11T16:08:28.905581', 'startedAt': '2024-12-11T16:08:28.497858', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '1b557f164c7f62a8d1545c761b13ede1677c3a83'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.98675, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.916, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.021, 'min': 800.0, 'max': 4700.0}, {'current': 4500.003, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 623.7553443908691}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71596908569336}} +2024-12-11 11:08:28,913 INFO HandlerThread:736906 [system_monitor.py:probe():224] Finished collecting system info +2024-12-11 11:08:28,913 INFO HandlerThread:736906 [system_monitor.py:probe():227] Publishing system info +2024-12-11 11:08:28,913 DEBUG HandlerThread:736906 [system_info.py:_save_code():44] Saving code +2024-12-11 11:08:28,915 DEBUG HandlerThread:736906 [system_info.py:_save_code():67] Saving code done +2024-12-11 11:08:28,915 DEBUG HandlerThread:736906 [system_info.py:_save_patches():84] Saving git patches +2024-12-11 11:08:28,980 DEBUG HandlerThread:736906 [system_info.py:_save_patches():126] Saving git patches done +2024-12-11 11:08:28,981 INFO HandlerThread:736906 [system_monitor.py:probe():229] Finished publishing system info +2024-12-11 11:08:28,984 DEBUG SenderThread:736906 [sender.py:send():379] send: files +2024-12-11 11:08:28,984 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-11 11:08:28,984 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-11 11:08:28,984 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-11 11:08:29,219 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: python_packages +2024-12-11 11:08:29,220 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: python_packages +2024-12-11 11:08:29,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:08:29,224 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:29,225 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:08:29,227 INFO HandlerThread:736906 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/tb" + save: true +} +control { + relay_id: "4910867fa5cc4067977e0abd74cdd7a1" +} +_info { + stream_id: "7k8jf8l5" +} + +2024-12-11 11:08:29,227 DEBUG HandlerThread:736906 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-11 11:08:29,233 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/x1l1f2rp-wandb-metadata.json +2024-12-11 11:08:29,271 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/i9e9567q-diff.patch +2024-12-11 11:08:29,293 DEBUG SenderThread:736906 [sender.py:send():379] send: telemetry +2024-12-11 11:08:29,293 DEBUG SenderThread:736906 [sender.py:send():379] send: tbrecord +2024-12-11 11:08:29,294 DEBUG SenderThread:736906 [sender.py:send():379] send: files +2024-12-11 11:08:29,294 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 with policy live +2024-12-11 11:08:29,297 INFO wandb-upload_1:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/rdhewrwt-code/FISH/eval_robot.py +2024-12-11 11:08:29,490 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/jpf62kj4-events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:08:29,814 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-metadata.json +2024-12-11 11:08:29,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/requirements.txt +2024-12-11 11:08:29,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/diff.patch +2024-12-11 11:08:29,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:29,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:08:29,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/code/FISH/eval_robot.py +2024-12-11 11:08:29,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/code +2024-12-11 11:08:29,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/code/FISH +2024-12-11 11:08:30,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:31,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:31,814 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:31,970 DEBUG SenderThread:736906 [sender.py:send():379] send: config +2024-12-11 11:08:32,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:33,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:33,814 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:34,030 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:08:34,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:35,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:35,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:36,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:37,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:37,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:38,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:39,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:08:39,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:40,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:41,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:41,817 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:42,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:43,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:44,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:08:44,220 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:08:44,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:44,316 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:08:45,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:45,818 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:46,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:47,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:48,223 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:49,223 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:49,344 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:08:49,819 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:50,223 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:51,223 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:51,819 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:52,223 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:53,224 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:53,820 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:54,224 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:54,424 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:08:55,224 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:55,820 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:56,224 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:57,224 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:57,821 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:58,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:59,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:08:59,220 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:08:59,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:59,437 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:08:59,822 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:59,822 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/config.yaml +2024-12-11 11:09:00,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:01,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:01,822 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:02,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:03,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:03,823 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:04,265 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:04,625 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:05,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:05,823 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:06,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:07,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:07,824 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:08,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:09,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:09,626 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:09,825 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:10,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:11,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:11,825 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:12,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:13,234 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:13,826 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:14,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:09:14,220 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:09:14,235 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:15,202 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:15,235 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:15,461 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:09:15,462 DEBUG SenderThread:736906 [sender.py:send():379] send: files +2024-12-11 11:09:15,463 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_a7ce057e6fe1a72a45a4.mp4 with policy now +2024-12-11 11:09:15,463 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:09:15,463 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:09:15,463 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:09:15,737 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/qmodnrad-media/videos/eval/0_eval_0_a7ce057e6fe1a72a45a4.mp4 +2024-12-11 11:09:15,826 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:15,827 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:09:15,827 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/0_eval_0_a7ce057e6fe1a72a45a4.mp4 +2024-12-11 11:09:15,827 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media +2024-12-11 11:09:15,827 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos +2024-12-11 11:09:15,827 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval +2024-12-11 11:09:16,102 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/9jxf4xwe-media/videos/eval/0_eval_0_a7ce057e6fe1a72a45a4.mp4 +2024-12-11 11:09:16,235 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:17,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:17,827 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:18,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:19,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:19,828 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:20,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:20,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:21,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:22,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:23,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:23,829 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:24,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:25,237 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:26,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:26,237 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:27,237 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:28,237 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:28,894 DEBUG SystemMonitor:736906 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-11 11:09:28,895 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:09:29,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:09:29,220 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:09:29,261 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:30,237 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:31,237 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:31,332 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:32,237 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:33,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:34,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:35,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:36,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:37,223 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:37,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:38,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:38,834 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:09:39,088 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/xszj87dx-events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:09:39,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:39,835 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:40,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:41,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:41,835 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:42,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:42,452 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:43,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:43,836 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:44,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:09:44,221 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:09:44,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:45,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:45,837 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:46,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:47,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:47,554 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:47,838 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:48,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:49,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:49,838 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:50,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:51,240 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:51,839 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:52,240 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:52,750 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:53,240 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:53,840 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:54,240 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:55,240 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:55,840 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:56,241 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:57,243 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:57,841 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:57,956 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:58,244 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:58,895 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:09:59,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:09:59,221 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:09:59,243 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:59,842 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:00,243 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:01,243 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:01,843 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:02,243 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:03,148 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:03,243 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:03,843 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:04,244 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:05,244 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:05,844 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:06,244 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:07,244 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:07,845 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:08,244 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:08,347 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:09,245 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:09,845 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:10,245 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:11,251 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:11,425 DEBUG SenderThread:736906 [sender.py:send():379] send: files +2024-12-11 11:10:11,425 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_dce37a039a0f3ae510e5.mp4 with policy now +2024-12-11 11:10:11,426 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:10:11,426 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:10:11,427 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:10:11,429 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:10:11,681 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/vbziq3qb-media/videos/eval/1_eval_1_dce37a039a0f3ae510e5.mp4 +2024-12-11 11:10:11,846 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:10:11,846 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:11,846 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/1_eval_1_dce37a039a0f3ae510e5.mp4 +2024-12-11 11:10:11,847 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval +2024-12-11 11:10:12,117 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/te2j2jvb-media/videos/eval/1_eval_1_dce37a039a0f3ae510e5.mp4 +2024-12-11 11:10:12,253 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:13,255 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:13,444 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:13,847 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:14,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:10:14,227 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:10:14,269 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:15,256 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:15,847 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:16,258 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:17,253 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:17,848 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:18,256 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:19,074 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:19,254 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:20,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:21,258 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:22,256 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:23,257 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:24,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:24,257 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:25,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:25,624 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:10:25,624 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:10:25,625 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:10:25,851 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:10:25,851 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:10:26,262 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:27,262 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:27,851 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:28,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:28,896 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:10:29,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:10:29,222 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:10:29,259 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:29,343 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:29,852 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:30,259 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:31,259 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:31,852 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:32,259 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:33,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:33,853 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:34,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:34,484 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:35,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:35,854 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:36,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:37,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:37,855 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:38,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:39,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:39,684 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:39,855 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:40,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:41,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:41,856 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:42,261 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:43,261 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:43,857 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:44,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:10:44,222 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:10:44,261 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:44,710 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:45,261 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:45,858 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:46,261 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:47,261 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:47,858 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:48,265 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:49,262 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:49,859 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:49,896 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:50,262 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:51,262 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:51,860 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:52,263 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:53,263 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:53,861 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:54,263 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:55,085 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:55,264 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:55,861 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:56,264 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:57,264 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:57,862 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:58,266 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:58,595 DEBUG SenderThread:736906 [sender.py:send():379] send: files +2024-12-11 11:10:58,595 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_c7cd658cf3e0e8f2dada.mp4 with policy now +2024-12-11 11:10:58,597 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:10:58,598 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:10:58,598 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:10:58,598 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:10:58,862 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:10:58,863 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/2_eval_2_c7cd658cf3e0e8f2dada.mp4 +2024-12-11 11:10:58,863 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval +2024-12-11 11:10:58,869 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/u7agmibs-media/videos/eval/2_eval_2_c7cd658cf3e0e8f2dada.mp4 +2024-12-11 11:10:58,896 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:10:59,153 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/veshwbiw-media/videos/eval/2_eval_2_c7cd658cf3e0e8f2dada.mp4 +2024-12-11 11:10:59,224 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:10:59,225 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:10:59,270 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:59,863 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:00,269 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:00,392 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:01,267 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:02,269 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:03,269 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:03,864 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:04,305 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:05,278 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:05,865 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:06,273 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:06,306 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:07,276 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:08,277 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:09,275 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:10,274 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:11,277 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:12,232 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:12,278 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:13,274 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:14,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:11:14,227 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:11:14,277 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:15,281 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:16,280 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:17,282 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:17,311 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:17,475 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:11:17,475 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:11:17,475 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:11:17,868 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:11:17,869 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:11:18,141 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/wagm2b1h-events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:11:18,283 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:19,288 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:19,869 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:20,287 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:21,285 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:21,870 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:22,285 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:22,473 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:23,282 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:23,871 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:24,301 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:25,284 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:25,871 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:26,282 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:27,292 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:27,477 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:27,872 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:28,283 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:28,897 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:11:29,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:11:29,227 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:11:29,283 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:29,873 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:30,285 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:31,284 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:31,873 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:32,284 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:32,506 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:33,286 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:33,874 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:34,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:35,288 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:35,875 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:36,291 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:37,291 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:37,511 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:37,875 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:38,289 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:39,288 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:39,876 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:40,290 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:41,290 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:41,877 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:42,291 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:42,593 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:43,292 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:43,878 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:44,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:11:44,226 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:11:44,293 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:45,292 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:45,880 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:46,292 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:47,294 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:47,631 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:47,880 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:48,293 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:49,329 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:49,881 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:50,298 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:51,303 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:51,882 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:52,314 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:52,331 DEBUG SenderThread:736906 [sender.py:send():379] send: files +2024-12-11 11:11:52,331 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_ac712fc970ab6ee59b1c.mp4 with policy now +2024-12-11 11:11:52,377 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:11:52,379 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:11:52,379 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:11:52,380 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:11:52,610 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/yfw8lnde-media/videos/eval/3_eval_3_ac712fc970ab6ee59b1c.mp4 +2024-12-11 11:11:52,883 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:11:52,883 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/3_eval_3_ac712fc970ab6ee59b1c.mp4 +2024-12-11 11:11:52,883 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval +2024-12-11 11:11:53,178 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/j207a1ci-media/videos/eval/3_eval_3_ac712fc970ab6ee59b1c.mp4 +2024-12-11 11:11:53,311 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:53,382 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:53,883 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:54,306 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:55,311 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:55,884 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:56,314 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:57,311 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:58,309 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:58,897 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:11:58,898 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:59,229 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:11:59,230 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:11:59,316 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:59,885 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:00,317 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:01,318 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:01,886 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:12:02,316 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:02,525 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:12:02,526 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:12:02,526 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:12:02,886 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:12:03,325 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:03,886 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:04,243 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:04,323 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:05,322 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:06,324 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:07,322 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:07,888 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:08,320 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:09,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:09,355 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:09,888 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:10,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:11,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:11,889 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:12,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:13,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:13,890 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:14,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:12:14,227 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:12:14,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:14,382 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:15,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:15,891 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:16,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:17,322 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:17,892 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:18,323 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:19,322 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:19,567 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:19,892 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:20,326 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:21,326 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:21,893 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:22,323 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:23,323 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:23,894 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:24,323 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:24,575 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:25,324 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:25,894 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:26,324 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:27,325 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:27,895 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:28,325 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:28,898 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:12:29,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:12:29,227 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:12:29,325 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:29,783 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:29,896 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:30,325 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:31,326 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:31,896 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:32,326 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:33,326 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:33,897 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:34,326 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:34,978 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:35,326 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:35,898 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:36,334 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:36,910 DEBUG SenderThread:736906 [sender.py:send():379] send: files +2024-12-11 11:12:36,910 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_b9f6613e7cd6916b7519.mp4 with policy now +2024-12-11 11:12:36,913 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:12:36,913 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:12:36,913 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:12:36,914 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:12:37,174 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/5dcy7vyd-media/videos/eval/4_eval_4_b9f6613e7cd6916b7519.mp4 +2024-12-11 11:12:37,337 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:37,899 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:12:37,899 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:37,899 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/4_eval_4_b9f6613e7cd6916b7519.mp4 +2024-12-11 11:12:37,899 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval +2024-12-11 11:12:38,192 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/663gsrmo-media/videos/eval/4_eval_4_b9f6613e7cd6916b7519.mp4 +2024-12-11 11:12:38,346 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:39,333 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:40,341 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:40,890 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:41,334 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:41,900 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:42,334 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:43,344 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:43,901 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:44,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:12:44,239 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:12:44,344 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:45,348 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:45,454 DEBUG SenderThread:736906 [sender.py:send():379] send: exit +2024-12-11 11:12:45,454 INFO SenderThread:736906 [sender.py:send_exit():586] handling exit code: 0 +2024-12-11 11:12:45,454 INFO SenderThread:736906 [sender.py:send_exit():588] handling runtime: 256 +2024-12-11 11:12:45,488 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:12:45,489 INFO SenderThread:736906 [sender.py:send_exit():594] send defer +2024-12-11 11:12:45,489 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:45,489 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-11 11:12:45,489 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:45,489 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-11 11:12:45,489 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 1 +2024-12-11 11:12:45,489 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:45,489 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-11 11:12:45,489 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:45,489 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-11 11:12:45,489 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 2 +2024-12-11 11:12:45,489 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:45,489 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-11 11:12:45,489 INFO HandlerThread:736906 [system_monitor.py:finish():203] Stopping system monitor +2024-12-11 11:12:45,490 DEBUG SystemMonitor:736906 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-11 11:12:45,490 DEBUG SystemMonitor:736906 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-11 11:12:45,490 INFO HandlerThread:736906 [interfaces.py:finish():200] Joined cpu monitor +2024-12-11 11:12:45,490 INFO HandlerThread:736906 [interfaces.py:finish():200] Joined disk monitor +2024-12-11 11:12:45,539 INFO HandlerThread:736906 [interfaces.py:finish():200] Joined gpu monitor +2024-12-11 11:12:45,540 INFO HandlerThread:736906 [interfaces.py:finish():200] Joined memory monitor +2024-12-11 11:12:45,540 INFO HandlerThread:736906 [interfaces.py:finish():200] Joined network monitor +2024-12-11 11:12:45,540 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:45,540 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-11 11:12:45,540 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 3 +2024-12-11 11:12:45,540 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:12:45,540 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:45,541 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-11 11:12:45,541 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:45,541 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-11 11:12:45,541 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 4 +2024-12-11 11:12:45,541 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:45,541 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-11 11:12:45,901 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:12:45,902 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:45,902 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:12:46,159 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/f6t033zr-events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:12:51,543 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:12:51,543 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:51,543 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:51,543 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: keepalive +2024-12-11 11:12:51,543 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-11 11:12:51,543 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:51,543 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 5 +2024-12-11 11:12:51,544 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:12:51,544 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:51,544 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:12:51,545 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-11 11:12:51,545 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:12:51,546 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:12:51,546 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:12:51,546 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:12:51,549 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:12:51,550 DEBUG SenderThread:736906 [sender.py:send():379] send: summary +2024-12-11 11:12:51,550 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:12:51,550 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:51,550 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-11 11:12:51,550 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 6 +2024-12-11 11:12:51,550 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:51,550 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-11 11:12:51,550 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:51,550 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-11 11:12:51,550 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 7 +2024-12-11 11:12:51,551 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:51,551 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:51,551 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-11 11:12:51,551 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:51,551 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-11 11:12:51,903 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:12:52,455 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:12:53,512 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 8 +2024-12-11 11:12:53,512 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:12:53,512 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:53,512 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-11 11:12:53,512 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:53,512 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-11 11:12:53,512 INFO SenderThread:736906 [job_builder.py:build():440] Attempting to build job artifact +2024-12-11 11:12:53,515 INFO SenderThread:736906 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-11 11:12:53,530 INFO SenderThread:736906 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-11 11:12:53,545 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 9 +2024-12-11 11:12:53,545 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:53,545 DEBUG SenderThread:736906 [sender.py:send():379] send: artifact +2024-12-11 11:12:53,545 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-11 11:12:53,904 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:54,052 INFO wandb-upload_1:736906 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpym6m9k3x +2024-12-11 11:12:54,052 INFO wandb-upload_2:736906 [upload_job.py:push():85] Skipped uploading /tmp/tmpq1auzwlr/wandb-job.json +2024-12-11 11:12:54,169 INFO wandb-upload_0:736906 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp9l1c58z3 +2024-12-11 11:12:54,456 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:12:54,800 INFO SenderThread:736906 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM2NzU4MjI2NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM2NzU3NzcyOA==', 'versionIndex': 360}}} +2024-12-11 11:12:54,800 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:54,800 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-11 11:12:54,800 INFO SenderThread:736906 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-11 11:12:54,905 INFO SenderThread:736906 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files +2024-12-11 11:12:54,905 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/config.yaml config.yaml +2024-12-11 11:12:54,905 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json wandb-summary.json +2024-12-11 11:12:54,905 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/requirements.txt requirements.txt +2024-12-11 11:12:54,906 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-metadata.json wandb-metadata.json +2024-12-11 11:12:54,908 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:12:54,909 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log output.log +2024-12-11 11:12:54,910 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/diff.patch diff.patch +2024-12-11 11:12:54,911 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-11 11:12:54,911 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/1_eval_1_dce37a039a0f3ae510e5.mp4 media/videos/eval/1_eval_1_dce37a039a0f3ae510e5.mp4 +2024-12-11 11:12:54,911 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/2_eval_2_c7cd658cf3e0e8f2dada.mp4 media/videos/eval/2_eval_2_c7cd658cf3e0e8f2dada.mp4 +2024-12-11 11:12:54,912 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/4_eval_4_b9f6613e7cd6916b7519.mp4 media/videos/eval/4_eval_4_b9f6613e7cd6916b7519.mp4 +2024-12-11 11:12:54,912 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/0_eval_0_a7ce057e6fe1a72a45a4.mp4 media/videos/eval/0_eval_0_a7ce057e6fe1a72a45a4.mp4 +2024-12-11 11:12:54,912 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/3_eval_3_ac712fc970ab6ee59b1c.mp4 media/videos/eval/3_eval_3_ac712fc970ab6ee59b1c.mp4 +2024-12-11 11:12:54,912 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 10 +2024-12-11 11:12:54,912 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:12:54,912 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:54,912 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-11 11:12:54,912 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:54,912 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-11 11:12:54,912 INFO SenderThread:736906 [file_pusher.py:finish():169] shutting down file pusher +2024-12-11 11:12:55,091 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/requirements.txt +2024-12-11 11:12:55,126 INFO wandb-upload_1:736906 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/config.yaml +2024-12-11 11:12:55,127 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:12:55,143 INFO wandb-upload_3:736906 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:55,344 INFO Thread-11 (_thread_body):736906 [sender.py:transition_state():614] send defer: 11 +2024-12-11 11:12:55,344 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:55,344 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-11 11:12:55,344 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:55,344 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-11 11:12:55,344 INFO SenderThread:736906 [file_pusher.py:join():175] waiting for file pusher +2024-12-11 11:12:55,345 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 12 +2024-12-11 11:12:55,345 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:55,345 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-11 11:12:55,345 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:55,345 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-11 11:12:55,346 INFO SenderThread:736906 [file_stream.py:finish():601] file stream finish called +2024-12-11 11:12:55,456 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:12:55,571 INFO SenderThread:736906 [file_stream.py:finish():605] file stream finish is done +2024-12-11 11:12:55,571 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 13 +2024-12-11 11:12:55,571 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:12:55,572 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:55,572 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-11 11:12:55,572 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:55,572 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-11 11:12:55,572 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 14 +2024-12-11 11:12:55,572 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:55,572 DEBUG SenderThread:736906 [sender.py:send():379] send: final +2024-12-11 11:12:55,572 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-11 11:12:55,572 DEBUG SenderThread:736906 [sender.py:send():379] send: footer +2024-12-11 11:12:55,572 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:55,573 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-11 11:12:55,573 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:12:55,573 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:12:55,573 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: server_info +2024-12-11 11:12:55,573 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: server_info +2024-12-11 11:12:55,575 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:12:55,575 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: get_summary +2024-12-11 11:12:55,575 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-11 11:12:55,576 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:55,652 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:12:55,652 INFO MainThread:736906 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-11 11:12:55,652 INFO MainThread:736906 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-11 11:12:55,653 INFO MainThread:736906 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-11 11:12:55,653 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: shutdown +2024-12-11 11:12:55,653 INFO HandlerThread:736906 [handler.py:finish():882] shutting down handler +2024-12-11 11:12:56,575 INFO WriterThread:736906 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/run-7k8jf8l5.wandb +2024-12-11 11:12:56,652 INFO SenderThread:736906 [sender.py:finish():1615] shutting down sender +2024-12-11 11:12:56,652 INFO SenderThread:736906 [file_pusher.py:finish():169] shutting down file pusher +2024-12-11 11:12:56,652 INFO SenderThread:736906 [file_pusher.py:join():175] waiting for file pusher diff --git a/110822/wandb/run-20241211_110828-7k8jf8l5/logs/debug.log b/110822/wandb/run-20241211_110828-7k8jf8l5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..dfce6deda856694095bb5922d4b33cde2714f342 --- /dev/null +++ b/110822/wandb/run-20241211_110828-7k8jf8l5/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-11 11:08:28,499 INFO MainThread:736760 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-11 11:08:28,499 INFO MainThread:736760 [wandb_setup.py:_flush():76] Configure stats pid to 736760 +2024-12-11 11:08:28,499 INFO MainThread:736760 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-11 11:08:28,499 INFO MainThread:736760 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/settings +2024-12-11 11:08:28,500 INFO MainThread:736760 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-11 11:08:28,500 INFO MainThread:736760 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-11 11:08:28,500 INFO MainThread:736760 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-11 11:08:28,500 INFO MainThread:736760 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-11 11:08:28,500 INFO MainThread:736760 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/logs/debug.log +2024-12-11 11:08:28,500 INFO MainThread:736760 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/logs/debug-internal.log +2024-12-11 11:08:28,500 INFO MainThread:736760 [wandb_init.py:init():569] calling init triggers +2024-12-11 11:08:28,500 INFO MainThread:736760 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '265644', 'wandb_notes': '265644_req_265752', 'eval': True, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim_left_to_right'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 16, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/265644', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'obs_shape': [5, 240, 320], 'action_shape': [36, 7], 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'true_action_history': False, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt'} +2024-12-11 11:08:28,500 INFO MainThread:736760 [wandb_init.py:init():619] starting backend +2024-12-11 11:08:28,500 INFO MainThread:736760 [wandb_init.py:init():623] setting up manager +2024-12-11 11:08:28,504 INFO MainThread:736760 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-11 11:08:28,505 INFO MainThread:736760 [wandb_init.py:init():631] backend started and connected +2024-12-11 11:08:28,525 INFO MainThread:736760 [wandb_init.py:init():720] updated telemetry +2024-12-11 11:08:28,531 INFO MainThread:736760 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-11 11:08:28,832 INFO MainThread:736760 [wandb_run.py:_on_init():2435] communicating current version +2024-12-11 11:08:28,875 INFO MainThread:736760 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.0 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-11 11:08:28,876 INFO MainThread:736760 [wandb_init.py:init():804] starting run threads in backend +2024-12-11 11:08:29,220 INFO MainThread:736760 [wandb_run.py:_console_start():2413] atexit reg +2024-12-11 11:08:29,224 INFO MainThread:736760 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-11 11:08:29,224 INFO MainThread:736760 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-11 11:08:29,225 INFO MainThread:736760 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-11 11:08:29,226 INFO MainThread:736760 [wandb_init.py:init():847] run started, returning control to user process +2024-12-11 11:08:29,227 INFO MainThread:736760 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/tb, True +2024-12-11 11:08:31,969 INFO MainThread:736760 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-11 11:12:56,656 WARNING MsgRouterThr:736760 [router.py:message_loop():77] message_loop has been closed diff --git a/110822/wandb/run-20241211_110828-7k8jf8l5/run-7k8jf8l5.wandb b/110822/wandb/run-20241211_110828-7k8jf8l5/run-7k8jf8l5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..d42487f0d0d4f01209404000598861558427348d --- /dev/null +++ b/110822/wandb/run-20241211_110828-7k8jf8l5/run-7k8jf8l5.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4546af021d2cd6549618885aaebb7421a983ca7bbd6e8d2e76f4c1222dec616b +size 290358 diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8411002a819f9e388787a4dbb369b9e1acc6349f --- /dev/null +++ b/config.yaml @@ -0,0 +1,453 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 0 +dataset_shuffle_seed: 1 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '265644' +wandb_notes: 265644_req_265752 +eval: false +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: +- hbm_twodim_left_to_right +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 16 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + obs_shape: + - 5 + - 240 + - 320 + action_shape: + - 36 + - 7 + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + policy_frequency: 5 + interpolation_frequency: 25 +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/snapshot_10500.pt b/snapshot_10500.pt new file mode 100644 index 0000000000000000000000000000000000000000..015f0d5aae1554e8238f6e71aeb3561073573838 --- /dev/null +++ b/snapshot_10500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed5151227c9bfbb1ce8651223449c640bbdc797f3f4605d88492631eb6912a34 +size 910102870 diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..a4fd203f3959578e5c449c9985eab2fbeb146b94 --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fae41135dec291bfcdac82db47fa9fb40a09e1ac76061e2a39370b132b782aa8 +size 910102870 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..16b4c8216e11e0f1ea8566c95915e1863992dd23 --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e993ca043fc19bb92114a39b5acb9f2d56dc068de8dbe8cd749abaed85c2eca +size 910102870 diff --git a/snapshot_1500.pt b/snapshot_1500.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f607b9ab846e72fced95446c9fb837df56cc28e --- /dev/null +++ b/snapshot_1500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:146eedb9ebca04c7d0581ff35af4a7937b362f8207f2296d5ebdc09c41699ce7 +size 910102360 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..962dcc6071a29eae5b6a45683cc72451d3c76f1e --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c79081eed6ee49d492dba6bafc7e030cba78907a50bf9f3be9be7a6467b9b44 +size 910102870 diff --git a/snapshot_2499.pt b/snapshot_2499.pt new file mode 100644 index 0000000000000000000000000000000000000000..4cf05cf6d7b8e3581b1d7de37edd118a5e59a04b --- /dev/null +++ b/snapshot_2499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3ddbdf2a62c3ad3212411856cffea983a8adaa9210dc015ec6425b997c4e6dd +size 910102360 diff --git a/snapshot_3000.pt b/snapshot_3000.pt new file mode 100644 index 0000000000000000000000000000000000000000..3e0a22b852f3a60fd1617b123045628bbae35fbe --- /dev/null +++ b/snapshot_3000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11a6a4c0caccc0c328a46523233bff856c3d82befeeb345a93e818e271f7aeff +size 910102360 diff --git a/snapshot_3499.pt b/snapshot_3499.pt new file mode 100644 index 0000000000000000000000000000000000000000..bea0edd43ccbc3e5af75405bb61e2e0daf0f646e --- /dev/null +++ b/snapshot_3499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fb186b9dac789a2e1f6c6935ddf8a4ac798485c52ad98428b2f3743f2404481 +size 910102360 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..089b91e78eddca1e6153c6c13b358de39d46c67e --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd653823b1da7bae744ee81756ec2f6e6e9076f3f328c82e16b5735bf76a9de5 +size 910102360 diff --git a/snapshot_4749.pt b/snapshot_4749.pt new file mode 100644 index 0000000000000000000000000000000000000000..f9bc7fa4ff6ff66ac682c289dddf7e4a3767b17d --- /dev/null +++ b/snapshot_4749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49af37b3b938a6758e8b678054e7381d58f172a27aab7e416abeac490a5ab1cc +size 910102360 diff --git a/snapshot_5749.pt b/snapshot_5749.pt new file mode 100644 index 0000000000000000000000000000000000000000..a35168ce5b1272f7d075b184136bbee782f2a7b6 --- /dev/null +++ b/snapshot_5749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84b71cad42523a9b806c065a63f56115bd4ab56af54bc4848b00ac247aac1a4f +size 910102360 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..e479555b05ec28426e0a3bee7a667e588627d25f --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:416344d4ce70215a5642ed6e9947a60b6d6ce5c900c2ed8035ed13f634a4a82f +size 910102360 diff --git a/snapshot_6749.pt b/snapshot_6749.pt new file mode 100644 index 0000000000000000000000000000000000000000..3d21cabde8e3c31cbb5bef4d343901054a597f65 --- /dev/null +++ b/snapshot_6749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f92a6ba981dcc900c7274cb00484768ef1ea6db30ea35eb657c26a20656a1cf +size 910102360 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..d58124f5729e3c5cb3250acc5f9dd83ffc3e94a9 --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb69c983cab49a43eeddba84547fe9c9df644d16e320457c519d8bf2b1a61638 +size 910102360 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..25a3876015e0e1b9697cb784c1e04d54a73b44a3 --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cdb0ee5ead08fdc6266a54270babc33bb6afe6d764ed6f808576ffcaa418449 +size 910102360 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..962dcc6071a29eae5b6a45683cc72451d3c76f1e --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c79081eed6ee49d492dba6bafc7e030cba78907a50bf9f3be9be7a6467b9b44 +size 910102870