diff --git a/.gitattributes b/.gitattributes index 2855ce2bcbadedbfe6ffa2f0b558817b7cb0014e..1c31437cce0ddef57cc997564a4321c1bcd705ef 100644 --- a/.gitattributes +++ b/.gitattributes @@ -53,3 +53,23 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text 214032/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text 214032/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text 214032/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +205324/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +205324/wandb/run-20241210_205330-52nddla7/files/media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4 filter=lfs diff=lfs merge=lfs -text +210407/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +210407/wandb/run-20241210_210414-ob7chcle/files/media/videos/eval/0_eval_0_242331cb3a64f66021df.mp4 filter=lfs diff=lfs merge=lfs -text +214032/episode_rosbags/episode_3_2024-12-10-21-43-33.rrd filter=lfs diff=lfs merge=lfs -text +210732/wandb/run-20241210_210738-r4m9fzdi/run-r4m9fzdi.wandb filter=lfs diff=lfs merge=lfs -text +210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/3_eval_3_49c63cb7694dee4c930a.mp4 filter=lfs diff=lfs merge=lfs -text +210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/2_eval_2_b17305aedebc761a4898.mp4 filter=lfs diff=lfs merge=lfs -text +210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/0_eval_0_daf03ac4df433b6afa80.mp4 filter=lfs diff=lfs merge=lfs -text +210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/1_eval_1_fa72f11620e871983c64.mp4 filter=lfs diff=lfs merge=lfs -text +210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/4_eval_4_a244b0b09dc15d2eae1c.mp4 filter=lfs diff=lfs merge=lfs -text +210732/episode_rosbags/episode_0_2024-12-10-21-08-09.rrd filter=lfs diff=lfs merge=lfs -text +210732/episode_rosbags/episode_4_2024-12-10-21-11-07.rrd filter=lfs diff=lfs merge=lfs -text +210732/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +210732/episode_rosbags/episode_2_2024-12-10-21-09-45.rrd filter=lfs diff=lfs merge=lfs -text +210732/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +210732/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +210732/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +210732/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +210732/episode_rosbags/episode_1_2024-12-10-21-09-04.rrd filter=lfs diff=lfs merge=lfs -text diff --git a/204951/.hydra/config.yaml b/204951/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..800e8f2818ebbf3474b49aa980471298b50b023c --- /dev/null +++ b/204951/.hydra/config.yaml @@ -0,0 +1,349 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '265626' +true_action_history: true +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/204951/.hydra/hydra.yaml b/204951/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e2c55aaaf76e07c0b4610dbef35ca5274850193 --- /dev/null +++ b/204951/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${experiment_dir}/${now:%H%M%S} + sweep: + dir: ${experiment_dir}/${now:%H%M%S} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${experiment_dir}/${now:%H%M%S}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/204951 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/204951/.hydra/overrides.yaml b/204951/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/204951/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/204951/eval_robot.log b/204951/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..98f41ce03ff75c4d9a711caff0fd1cc69b82a0c8 --- /dev/null +++ b/204951/eval_robot.log @@ -0,0 +1,13 @@ +[2024-12-10 20:49:51,284][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-10 20:49:51,286][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2024-12-10 20:49:51,984][rospy.core][INFO] - signal_shutdown [atexit] diff --git a/205307/.hydra/overrides.yaml b/205307/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/205307/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/205324/eval_video/0_eval.mp4 b/205324/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0e1266f34af4bbc626a14073fe2213e8026daa9f --- /dev/null +++ b/205324/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff4edecb6a17ea039fbf12ba8ab1396b6c7798607c09ee107432df34eedac9aa +size 622320 diff --git a/205324/tb/events.out.tfevents.1733882011.leonmkim-ROG-Strix-G15CS-G15CS.597884.0 b/205324/tb/events.out.tfevents.1733882011.leonmkim-ROG-Strix-G15CS-G15CS.597884.0 new file mode 100644 index 0000000000000000000000000000000000000000..d8c5949c7a49f3e3c6a62e70e68137af68e51e0e --- /dev/null +++ b/205324/tb/events.out.tfevents.1733882011.leonmkim-ROG-Strix-G15CS-G15CS.597884.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e8967cbc6351092e6da6d4bc739212242a2482a1b91b1d78242392a4546cb4a +size 88 diff --git a/205324/wandb/run-20241210_205330-52nddla7/files/media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4 b/205324/wandb/run-20241210_205330-52nddla7/files/media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0e1266f34af4bbc626a14073fe2213e8026daa9f --- /dev/null +++ b/205324/wandb/run-20241210_205330-52nddla7/files/media/videos/eval/video_0_ff4edecb6a17ea039fbf.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff4edecb6a17ea039fbf12ba8ab1396b6c7798607c09ee107432df34eedac9aa +size 622320 diff --git a/210407/.hydra/config.yaml b/210407/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d37c497bc2fe8883c44596565f059ac8e1269416 --- /dev/null +++ b/210407/.hydra/config.yaml @@ -0,0 +1,349 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '265626' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/210407/.hydra/hydra.yaml b/210407/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eaf84cfd52ac520bea093126b8da4d7805f8737b --- /dev/null +++ b/210407/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${experiment_dir}/${now:%H%M%S} + sweep: + dir: ${experiment_dir}/${now:%H%M%S} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${experiment_dir}/${now:%H%M%S}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/210407/.hydra/overrides.yaml b/210407/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/210407/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/210407/episode_rosbags/aligned_depth_to_color_K.npy b/210407/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/210407/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/210407/episode_rosbags/cam_tf_world.npy b/210407/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/210407/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/210407/episode_rosbags/color_K.npy b/210407/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/210407/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/210407/episode_rosbags/depth_K.npy b/210407/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/210407/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/210407/eval_robot.log b/210407/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..c81de3ae28ad4b6a6d0e254da0d9830460ea33d0 --- /dev/null +++ b/210407/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-10 21:04:07,944][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-10 21:04:07,947][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/210407/eval_video/0_eval.mp4 b/210407/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5ae81c700a986d6390e22d5c7f44c214169b40e5 --- /dev/null +++ b/210407/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:242331cb3a64f66021dfad95eeae9d0d8610c587d62363724138c291f7e496ce +size 1091529 diff --git a/210407/eval_video/1_eval.mp4 b/210407/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa868fbb75c51fb59ae42ff30716054804074343 Binary files /dev/null and b/210407/eval_video/1_eval.mp4 differ diff --git a/210407/tb/events.out.tfevents.1733882654.leonmkim-ROG-Strix-G15CS-G15CS.599894.0 b/210407/tb/events.out.tfevents.1733882654.leonmkim-ROG-Strix-G15CS-G15CS.599894.0 new file mode 100644 index 0000000000000000000000000000000000000000..0f444d57807a5b4dca48da9a30be34f1c636742b --- /dev/null +++ b/210407/tb/events.out.tfevents.1733882654.leonmkim-ROG-Strix-G15CS-G15CS.599894.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36accc6becaa391cb3a6521f7d1f74758e9ca695b2f1d586c08562ee2453d0ec +size 88 diff --git a/210407/wandb/debug-internal.log b/210407/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..dd0fc984e9385549fd72db7ed4ef7b2dfdb78f56 --- /dev/null +++ b/210407/wandb/debug-internal.log @@ -0,0 +1,510 @@ +2024-12-10 21:04:14,256 INFO StreamThr :600018 [internal.py:wandb_internal():85] W&B internal server running at pid: 600018, started at: 2024-12-10 21:04:14.256478 +2024-12-10 21:04:14,265 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status +2024-12-10 21:04:14,269 INFO WriterThread:600018 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/run-ob7chcle.wandb +2024-12-10 21:04:14,270 DEBUG SenderThread:600018 [sender.py:send():379] send: header +2024-12-10 21:04:14,286 DEBUG SenderThread:600018 [sender.py:send():379] send: run +2024-12-10 21:04:14,541 INFO SenderThread:600018 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files +2024-12-10 21:04:14,541 INFO SenderThread:600018 [sender.py:_start_run_threads():1188] run started: ob7chcle with start time 1733882654.256577 +2024-12-10 21:04:14,564 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: check_version +2024-12-10 21:04:14,564 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: check_version +2024-12-10 21:04:14,612 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: run_start +2024-12-10 21:04:14,617 DEBUG HandlerThread:600018 [system_info.py:__init__():26] System info init +2024-12-10 21:04:14,617 DEBUG HandlerThread:600018 [system_info.py:__init__():41] System info init done +2024-12-10 21:04:14,617 INFO HandlerThread:600018 [system_monitor.py:start():194] Starting system monitor +2024-12-10 21:04:14,617 INFO SystemMonitor:600018 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-10 21:04:14,617 INFO HandlerThread:600018 [system_monitor.py:probe():214] Collecting system info +2024-12-10 21:04:14,618 INFO SystemMonitor:600018 [interfaces.py:start():188] Started cpu monitoring +2024-12-10 21:04:14,618 INFO SystemMonitor:600018 [interfaces.py:start():188] Started disk monitoring +2024-12-10 21:04:14,619 INFO SystemMonitor:600018 [interfaces.py:start():188] Started gpu monitoring +2024-12-10 21:04:14,619 INFO SystemMonitor:600018 [interfaces.py:start():188] Started memory monitoring +2024-12-10 21:04:14,620 INFO SystemMonitor:600018 [interfaces.py:start():188] Started network monitoring +2024-12-10 21:04:14,630 DEBUG HandlerThread:600018 [system_info.py:probe():152] Probing system +2024-12-10 21:04:14,631 DEBUG HandlerThread:600018 [system_info.py:_probe_git():137] Probing git +2024-12-10 21:04:14,636 DEBUG HandlerThread:600018 [system_info.py:_probe_git():145] Probing git done +2024-12-10 21:04:14,636 DEBUG HandlerThread:600018 [system_info.py:probe():200] Probing system done +2024-12-10 21:04:14,636 DEBUG HandlerThread:600018 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-11T02:04:14.630135', 'startedAt': '2024-12-11T02:04:14.247012', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '4f7c0421eb84a7caf6bf5670f68777523362676f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4124.996375000001, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.021, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}, {'current': 4500.023, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 366.0494689941406}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71596908569336}} +2024-12-10 21:04:14,636 INFO HandlerThread:600018 [system_monitor.py:probe():224] Finished collecting system info +2024-12-10 21:04:14,636 INFO HandlerThread:600018 [system_monitor.py:probe():227] Publishing system info +2024-12-10 21:04:14,636 DEBUG HandlerThread:600018 [system_info.py:_save_code():44] Saving code +2024-12-10 21:04:14,638 DEBUG HandlerThread:600018 [system_info.py:_save_code():67] Saving code done +2024-12-10 21:04:14,638 DEBUG HandlerThread:600018 [system_info.py:_save_patches():84] Saving git patches +2024-12-10 21:04:14,703 DEBUG HandlerThread:600018 [system_info.py:_save_patches():126] Saving git patches done +2024-12-10 21:04:14,704 INFO HandlerThread:600018 [system_monitor.py:probe():229] Finished publishing system info +2024-12-10 21:04:14,707 DEBUG SenderThread:600018 [sender.py:send():379] send: files +2024-12-10 21:04:14,707 INFO SenderThread:600018 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-10 21:04:14,708 INFO SenderThread:600018 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-10 21:04:14,708 INFO SenderThread:600018 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-10 21:04:14,959 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: python_packages +2024-12-10 21:04:14,960 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: python_packages +2024-12-10 21:04:14,960 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:04:14,960 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:04:14,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:14,963 INFO HandlerThread:600018 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/tb" + save: true +} +control { + relay_id: "5cdf21b441a841eeb077b1d066a521dc" +} +_info { + stream_id: "ob7chcle" +} + +2024-12-10 21:04:14,963 DEBUG HandlerThread:600018 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-10 21:04:14,988 INFO wandb-upload_0:600018 [upload_job.py:push():130] Uploaded file /tmp/tmpr0qlu59lwandb/vuiipzpx-wandb-metadata.json +2024-12-10 21:04:15,027 DEBUG SenderThread:600018 [sender.py:send():379] send: telemetry +2024-12-10 21:04:15,027 DEBUG SenderThread:600018 [sender.py:send():379] send: tbrecord +2024-12-10 21:04:15,028 DEBUG SenderThread:600018 [sender.py:send():379] send: files +2024-12-10 21:04:15,028 INFO SenderThread:600018 [sender.py:_save_file():1454] saving file events.out.tfevents.1733882654.leonmkim-ROG-Strix-G15CS-G15CS.599894.0 with policy live +2024-12-10 21:04:15,030 INFO wandb-upload_1:600018 [upload_job.py:push():130] Uploaded file /tmp/tmpr0qlu59lwandb/4jll3ix3-code/FISH/eval_robot.py +2024-12-10 21:04:15,039 INFO wandb-upload_2:600018 [upload_job.py:push():130] Uploaded file /tmp/tmpr0qlu59lwandb/5pcafv76-diff.patch +2024-12-10 21:04:15,220 INFO wandb-upload_0:600018 [upload_job.py:push():130] Uploaded file /tmp/tmpr0qlu59lwandb/o3h98mw2-events.out.tfevents.1733882654.leonmkim-ROG-Strix-G15CS-G15CS.599894.0 +2024-12-10 21:04:15,543 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/requirements.txt +2024-12-10 21:04:15,543 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/code/FISH/eval_robot.py +2024-12-10 21:04:15,543 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/wandb-metadata.json +2024-12-10 21:04:15,543 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:04:15,543 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/events.out.tfevents.1733882654.leonmkim-ROG-Strix-G15CS-G15CS.599894.0 +2024-12-10 21:04:15,543 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/diff.patch +2024-12-10 21:04:15,543 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/code/FISH +2024-12-10 21:04:15,544 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/code +2024-12-10 21:04:15,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:16,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:17,542 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:04:17,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:18,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:19,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:04:19,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:20,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:21,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:22,962 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:23,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:24,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:04:24,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:25,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:26,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:27,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:28,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:29,960 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:04:29,960 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:04:30,002 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:30,045 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:04:30,965 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:31,965 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:32,965 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:33,965 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:34,965 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:35,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:04:35,965 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:36,965 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:37,966 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:38,966 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:39,966 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:40,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:04:40,966 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:41,966 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:42,966 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:43,966 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:44,960 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:04:44,960 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:04:45,001 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:45,967 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:46,018 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:04:46,550 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/config.yaml +2024-12-10 21:04:46,967 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:47,967 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:48,967 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:49,967 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:50,967 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:51,962 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:04:51,967 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:52,968 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:53,968 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:54,968 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:55,968 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:56,962 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:04:56,968 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:57,968 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:58,968 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:59,960 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:04:59,961 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:05:00,002 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:00,969 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:01,969 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:02,107 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:05:02,969 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:03,969 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:04,969 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:05,969 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:06,970 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:07,962 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:05:07,970 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:08,018 DEBUG SenderThread:600018 [sender.py:send():379] send: config +2024-12-10 21:05:08,970 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:09,556 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:09,970 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:10,970 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:11,556 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:11,971 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:12,970 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:13,449 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:05:13,557 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:13,971 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:14,620 DEBUG SystemMonitor:600018 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-10 21:05:14,621 DEBUG SenderThread:600018 [sender.py:send():379] send: stats +2024-12-10 21:05:14,960 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:05:14,961 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:05:15,002 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:15,558 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:15,971 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:16,971 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:17,558 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:17,971 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:18,600 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:05:18,971 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:19,559 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:19,559 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/config.yaml +2024-12-10 21:05:19,971 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:20,971 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:21,559 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:21,971 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:22,971 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:23,560 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:23,971 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:24,064 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:05:24,972 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:25,560 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:25,972 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:26,972 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:27,561 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:27,972 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:28,973 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:29,124 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:05:29,562 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:29,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:05:29,961 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:05:29,973 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:30,973 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:31,562 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:31,973 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:32,975 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:33,563 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:33,975 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:34,275 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:05:34,975 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:35,563 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:35,976 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:36,976 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:37,564 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:37,976 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:38,976 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:39,457 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:05:39,564 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:39,976 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:40,976 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:41,565 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:41,977 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:42,977 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:43,566 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:43,977 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:44,621 DEBUG SenderThread:600018 [sender.py:send():379] send: stats +2024-12-10 21:05:44,622 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:05:44,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:05:44,961 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:05:44,977 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:45,567 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:45,977 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:46,977 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:47,567 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:47,977 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:48,977 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:49,568 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:49,648 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:05:49,880 DEBUG SenderThread:600018 [sender.py:send():379] send: files +2024-12-10 21:05:49,880 INFO SenderThread:600018 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_242331cb3a64f66021df.mp4 with policy now +2024-12-10 21:05:49,880 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:05:49,881 DEBUG SenderThread:600018 [sender.py:send():379] send: history +2024-12-10 21:05:49,881 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:05:49,882 INFO SenderThread:600018 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:05:49,978 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:50,165 INFO wandb-upload_1:600018 [upload_job.py:push():130] Uploaded file /tmp/tmpr0qlu59lwandb/dx8d7p0q-media/videos/eval/0_eval_0_242331cb3a64f66021df.mp4 +2024-12-10 21:05:50,573 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/media/videos/eval/0_eval_0_242331cb3a64f66021df.mp4 +2024-12-10 21:05:50,573 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/wandb-summary.json +2024-12-10 21:05:50,573 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/media/videos/eval +2024-12-10 21:05:50,573 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/media/videos +2024-12-10 21:05:50,574 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/media +2024-12-10 21:05:50,912 INFO wandb-upload_0:600018 [upload_job.py:push():130] Uploaded file /tmp/tmpr0qlu59lwandb/64whhgw0-media/videos/eval/0_eval_0_242331cb3a64f66021df.mp4 +2024-12-10 21:05:50,978 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:51,573 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:51,978 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:52,978 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:53,574 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:53,979 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:54,963 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:05:54,978 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:55,978 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:56,978 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:57,575 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:57,978 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:58,979 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:59,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:05:59,961 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:06:00,001 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:00,045 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:06:00,979 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:01,979 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:02,979 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:03,979 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:04,979 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:05,963 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:06:05,979 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:06,979 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:07,979 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:08,980 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:09,980 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:10,980 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:11,963 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:06:11,980 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:12,980 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:13,980 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:14,622 DEBUG SenderThread:600018 [sender.py:send():379] send: stats +2024-12-10 21:06:14,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:06:14,961 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:06:15,001 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:15,980 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:16,980 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:17,057 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:06:17,980 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:18,981 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:19,981 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:20,981 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:21,981 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:22,963 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:06:22,981 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:23,981 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:24,981 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:25,982 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:26,982 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:27,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:06:27,982 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:28,982 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:29,962 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:06:29,962 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:06:30,005 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:30,982 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:31,982 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:32,982 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:33,045 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:06:33,983 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:34,983 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:35,983 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:36,983 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:37,983 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:38,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:06:38,983 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:39,983 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:40,984 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:41,984 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:42,984 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:43,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:06:43,985 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:44,622 DEBUG SenderThread:600018 [sender.py:send():379] send: stats +2024-12-10 21:06:44,962 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:06:44,962 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:06:45,006 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:45,986 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:46,986 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:47,987 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:48,986 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:49,027 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:06:49,987 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:50,986 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:51,987 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:52,987 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:53,987 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:54,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:06:54,987 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:55,987 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:56,987 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:57,987 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:58,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:59,962 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:06:59,962 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:07:00,005 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:00,048 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:07:00,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:01,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:02,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:03,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:04,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:05,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:07:05,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:06,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:07,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:08,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:09,989 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:10,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:07:10,989 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:11,989 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:12,999 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:13,402 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:07:13,402 DEBUG SenderThread:600018 [sender.py:send():379] send: files +2024-12-10 21:07:13,402 INFO SenderThread:600018 [sender.py:_save_file():1454] saving file media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 with policy now +2024-12-10 21:07:13,403 DEBUG SenderThread:600018 [sender.py:send():379] send: history +2024-12-10 21:07:13,403 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:07:13,403 INFO SenderThread:600018 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:07:13,404 DEBUG SenderThread:600018 [sender.py:send():379] send: exit +2024-12-10 21:07:13,404 INFO SenderThread:600018 [sender.py:send_exit():586] handling exit code: 0 +2024-12-10 21:07:13,404 INFO SenderThread:600018 [sender.py:send_exit():588] handling runtime: 178 +2024-12-10 21:07:13,404 INFO SenderThread:600018 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:07:13,404 INFO SenderThread:600018 [sender.py:send_exit():594] send defer +2024-12-10 21:07:13,404 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:13,404 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-10 21:07:13,405 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:13,405 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-10 21:07:13,405 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 1 +2024-12-10 21:07:13,405 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:13,405 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-10 21:07:13,406 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:13,406 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-10 21:07:13,406 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 2 +2024-12-10 21:07:13,407 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:13,407 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-10 21:07:13,407 INFO HandlerThread:600018 [system_monitor.py:finish():203] Stopping system monitor +2024-12-10 21:07:13,407 DEBUG SystemMonitor:600018 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-10 21:07:13,407 DEBUG SystemMonitor:600018 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-10 21:07:13,407 INFO HandlerThread:600018 [interfaces.py:finish():200] Joined cpu monitor +2024-12-10 21:07:13,408 INFO HandlerThread:600018 [interfaces.py:finish():200] Joined disk monitor +2024-12-10 21:07:13,456 INFO HandlerThread:600018 [interfaces.py:finish():200] Joined gpu monitor +2024-12-10 21:07:13,456 INFO HandlerThread:600018 [interfaces.py:finish():200] Joined memory monitor +2024-12-10 21:07:13,456 INFO HandlerThread:600018 [interfaces.py:finish():200] Joined network monitor +2024-12-10 21:07:13,456 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:13,456 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-10 21:07:13,456 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 3 +2024-12-10 21:07:13,456 DEBUG SenderThread:600018 [sender.py:send():379] send: stats +2024-12-10 21:07:13,456 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:13,457 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-10 21:07:13,457 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:13,457 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-10 21:07:13,457 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 4 +2024-12-10 21:07:13,457 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:13,457 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-10 21:07:13,599 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:07:13,600 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/wandb-summary.json +2024-12-10 21:07:13,600 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 +2024-12-10 21:07:13,600 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/media/videos/eval +2024-12-10 21:07:13,607 INFO wandb-upload_1:600018 [upload_job.py:push():130] Uploaded file /tmp/tmpr0qlu59lwandb/ycl94f2h-media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 +2024-12-10 21:07:13,892 INFO wandb-upload_0:600018 [upload_job.py:push():130] Uploaded file /tmp/tmpr0qlu59lwandb/53aux0ik-media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 +2024-12-10 21:07:15,600 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:07:19,985 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:07:19,985 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:19,985 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:07:19,986 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-10 21:07:19,986 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: keepalive +2024-12-10 21:07:19,986 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 5 +2024-12-10 21:07:19,986 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:07:19,986 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:19,986 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-10 21:07:19,987 DEBUG SenderThread:600018 [sender.py:send():379] send: summary +2024-12-10 21:07:19,987 INFO SenderThread:600018 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:07:19,987 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:19,987 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-10 21:07:19,987 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 6 +2024-12-10 21:07:19,987 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:19,987 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-10 21:07:19,987 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:19,988 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-10 21:07:19,988 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 7 +2024-12-10 21:07:19,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:07:19,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:19,988 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-10 21:07:19,988 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:19,988 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-10 21:07:20,405 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:07:20,602 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/wandb-summary.json +2024-12-10 21:07:21,263 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 8 +2024-12-10 21:07:21,263 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:07:21,263 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:21,263 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-10 21:07:21,263 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:21,263 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-10 21:07:21,263 INFO SenderThread:600018 [job_builder.py:build():440] Attempting to build job artifact +2024-12-10 21:07:21,266 INFO SenderThread:600018 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-10 21:07:21,280 INFO SenderThread:600018 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-10 21:07:21,294 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 9 +2024-12-10 21:07:21,295 DEBUG SenderThread:600018 [sender.py:send():379] send: artifact +2024-12-10 21:07:21,295 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:21,296 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-10 21:07:21,405 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:07:21,602 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:07:21,765 INFO wandb-upload_2:600018 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0up_8ql4 +2024-12-10 21:07:21,896 INFO wandb-upload_1:600018 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmphpmz9ew4 +2024-12-10 21:07:21,956 INFO wandb-upload_0:600018 [upload_job.py:push():88] Uploaded file /tmp/tmpbv2i4zjr/wandb-job.json +2024-12-10 21:07:22,564 INFO SenderThread:600018 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM2Njc3NTA2NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM2Njc2MTczOA==', 'versionIndex': 337}}} +2024-12-10 21:07:22,564 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:22,564 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-10 21:07:22,564 INFO SenderThread:600018 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-10 21:07:22,603 INFO SenderThread:600018 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files +2024-12-10 21:07:22,603 INFO SenderThread:600018 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/config.yaml config.yaml +2024-12-10 21:07:22,603 INFO SenderThread:600018 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/wandb-summary.json wandb-summary.json +2024-12-10 21:07:22,605 INFO SenderThread:600018 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/requirements.txt requirements.txt +2024-12-10 21:07:22,605 INFO SenderThread:600018 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/wandb-metadata.json wandb-metadata.json +2024-12-10 21:07:22,606 INFO SenderThread:600018 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/events.out.tfevents.1733882654.leonmkim-ROG-Strix-G15CS-G15CS.599894.0 events.out.tfevents.1733882654.leonmkim-ROG-Strix-G15CS-G15CS.599894.0 +2024-12-10 21:07:22,606 INFO SenderThread:600018 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log output.log +2024-12-10 21:07:22,607 INFO SenderThread:600018 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/diff.patch diff.patch +2024-12-10 21:07:22,608 INFO SenderThread:600018 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-10 21:07:22,610 INFO SenderThread:600018 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/media/videos/eval/0_eval_0_242331cb3a64f66021df.mp4 media/videos/eval/0_eval_0_242331cb3a64f66021df.mp4 +2024-12-10 21:07:22,610 INFO SenderThread:600018 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 +2024-12-10 21:07:22,610 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 10 +2024-12-10 21:07:22,610 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:07:22,610 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:22,610 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-10 21:07:22,610 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:22,610 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-10 21:07:22,610 INFO SenderThread:600018 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 21:07:22,797 INFO wandb-upload_0:600018 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/wandb-summary.json +2024-12-10 21:07:22,818 INFO wandb-upload_1:600018 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/requirements.txt +2024-12-10 21:07:22,829 INFO wandb-upload_2:600018 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/config.yaml +2024-12-10 21:07:22,893 INFO wandb-upload_3:600018 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:07:23,093 INFO Thread-11 (_thread_body):600018 [sender.py:transition_state():614] send defer: 11 +2024-12-10 21:07:23,094 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:23,094 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-10 21:07:23,094 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:23,094 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-10 21:07:23,094 INFO SenderThread:600018 [file_pusher.py:join():175] waiting for file pusher +2024-12-10 21:07:23,094 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 12 +2024-12-10 21:07:23,095 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:23,095 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-10 21:07:23,095 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:23,095 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-10 21:07:23,095 INFO SenderThread:600018 [file_stream.py:finish():601] file stream finish called +2024-12-10 21:07:23,318 INFO SenderThread:600018 [file_stream.py:finish():605] file stream finish is done +2024-12-10 21:07:23,319 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 13 +2024-12-10 21:07:23,319 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:23,319 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-10 21:07:23,319 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:23,319 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-10 21:07:23,319 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 14 +2024-12-10 21:07:23,319 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:23,319 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-10 21:07:23,319 DEBUG SenderThread:600018 [sender.py:send():379] send: final +2024-12-10 21:07:23,320 DEBUG SenderThread:600018 [sender.py:send():379] send: footer +2024-12-10 21:07:23,320 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:23,320 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-10 21:07:23,320 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:07:23,320 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: server_info +2024-12-10 21:07:23,320 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:07:23,321 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: server_info +2024-12-10 21:07:23,322 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:07:23,322 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: get_summary +2024-12-10 21:07:23,323 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-10 21:07:23,323 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:23,446 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:07:23,446 INFO MainThread:600018 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-10 21:07:23,446 INFO MainThread:600018 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-10 21:07:23,446 INFO MainThread:600018 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-10 21:07:23,446 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: shutdown +2024-12-10 21:07:23,446 INFO HandlerThread:600018 [handler.py:finish():882] shutting down handler +2024-12-10 21:07:24,322 INFO WriterThread:600018 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/run-ob7chcle.wandb +2024-12-10 21:07:24,446 INFO SenderThread:600018 [sender.py:finish():1615] shutting down sender +2024-12-10 21:07:24,446 INFO SenderThread:600018 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 21:07:24,446 INFO SenderThread:600018 [file_pusher.py:join():175] waiting for file pusher diff --git a/210407/wandb/debug.log b/210407/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4b3ee6e78ce69391bc1cdea3a993b48122fc9d53 --- /dev/null +++ b/210407/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-10 21:04:14,250 INFO MainThread:599894 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-10 21:04:14,250 INFO MainThread:599894 [wandb_setup.py:_flush():76] Configure stats pid to 599894 +2024-12-10 21:04:14,250 INFO MainThread:599894 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-10 21:04:14,250 INFO MainThread:599894 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/settings +2024-12-10 21:04:14,250 INFO MainThread:599894 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-10 21:04:14,250 INFO MainThread:599894 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-10 21:04:14,250 INFO MainThread:599894 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-10 21:04:14,250 INFO MainThread:599894 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-10 21:04:14,250 INFO MainThread:599894 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/logs/debug.log +2024-12-10 21:04:14,251 INFO MainThread:599894 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/logs/debug-internal.log +2024-12-10 21:04:14,251 INFO MainThread:599894 [wandb_init.py:init():569] calling init triggers +2024-12-10 21:04:14,251 INFO MainThread:599894 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '265626', 'wandb_notes': '265626_req_265827', 'eval': True, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim_left_to_right'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'obs_shape': [5, 240, 320], 'action_shape': [36, 7], 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'true_action_history': False, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt'} +2024-12-10 21:04:14,251 INFO MainThread:599894 [wandb_init.py:init():619] starting backend +2024-12-10 21:04:14,251 INFO MainThread:599894 [wandb_init.py:init():623] setting up manager +2024-12-10 21:04:14,255 INFO MainThread:599894 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-10 21:04:14,256 INFO MainThread:599894 [wandb_init.py:init():631] backend started and connected +2024-12-10 21:04:14,268 INFO MainThread:599894 [wandb_init.py:init():720] updated telemetry +2024-12-10 21:04:14,283 INFO MainThread:599894 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-10 21:04:14,563 INFO MainThread:599894 [wandb_run.py:_on_init():2435] communicating current version +2024-12-10 21:04:14,605 INFO MainThread:599894 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.0 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-10 21:04:14,606 INFO MainThread:599894 [wandb_init.py:init():804] starting run threads in backend +2024-12-10 21:04:14,960 INFO MainThread:599894 [wandb_run.py:_console_start():2413] atexit reg +2024-12-10 21:04:14,960 INFO MainThread:599894 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-10 21:04:14,960 INFO MainThread:599894 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-10 21:04:14,961 INFO MainThread:599894 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-10 21:04:14,962 INFO MainThread:599894 [wandb_init.py:init():847] run started, returning control to user process +2024-12-10 21:04:14,962 INFO MainThread:599894 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/tb, True +2024-12-10 21:05:08,018 INFO MainThread:599894 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-10 21:07:24,447 WARNING MsgRouterThr:599894 [router.py:message_loop():77] message_loop has been closed diff --git a/210407/wandb/run-20241210_210414-ob7chcle/files/code/FISH/eval_robot.py b/210407/wandb/run-20241210_210414-ob7chcle/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..75a95b02db44bf1d6cdce4329109f52a0c4c1bc8 --- /dev/null +++ b/210407/wandb/run-20241210_210414-ob7chcle/files/code/FISH/eval_robot.py @@ -0,0 +1,512 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/210407/wandb/run-20241210_210414-ob7chcle/files/config.yaml b/210407/wandb/run-20241210_210414-ob7chcle/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df69ae956e209a86bb8b3dcdebfc8c8dc23992de --- /dev/null +++ b/210407/wandb/run-20241210_210414-ob7chcle/files/config.yaml @@ -0,0 +1,966 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '265626' +wandb_notes: + desc: null + value: 265626_req_265827 +eval: + desc: null + value: true +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: + - hbm_twodim_left_to_right +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 0 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/265626 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + obs_shape: + - 5 + - 240 + - 320 + action_shape: + - 36 + - 7 + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + policy_frequency: 5 + interpolation_frequency: 25 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + policy_frequency: 5 + interpolation_frequency: 25 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +true_action_history: + desc: null + value: false +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1733882654 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 16 + - 23 + - 35 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/210407/wandb/run-20241210_210414-ob7chcle/files/diff.patch b/210407/wandb/run-20241210_210414-ob7chcle/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..95470eecc8e7dc869cd1c413f78c2ec1449a71c5 --- /dev/null +++ b/210407/wandb/run-20241210_210414-ob7chcle/files/diff.patch @@ -0,0 +1,47 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a9568e3..aa52de5 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -123,6 +123,11 @@ bc_weight_type: 'qfilter' # linear, qfilter + + # Load weights + load_checkpoint: ${agent.load_checkpoint} ++# 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act ++# full dataset ++wandb_run_id: "265626" # RGBD+Mask+act history ++# wandb_run_id: "265628" # RGBD+Mask+contact+act history ++ + # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # full dataset + # 6 step action history +@@ -135,7 +140,7 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '264772_0' # RGBD+act history + # wandb_run_id: '264775_0' # RGBD+mask+act history + # wandb_run_id: '264776_0' # RGBD+contact+act history +-wandb_run_id: '264777_0' # RGBD+mask+contact+act history ++# wandb_run_id: '264777_0' # RGBD+mask+contact+act history + + # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 16 held out demos +@@ -151,7 +156,7 @@ wandb_run_id: '264777_0' # RGBD+mask+contact+act history + # wandb_run_id: '250380_0' # RGBD+mask+contact+act history + # wandb_run_id: '250370_0' # RGBD+contact+act history + +-true_action_history: true ++true_action_history: false + wandb_notes: null + + checkpoint_epoch: 12000 # for depth + all contact +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 26f8ae3..4536380 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -627,7 +627,7 @@ class Franka: + # # append the gripper actions to the end of the 6d actions + # plan_dict['action_trajectory'] = np.concatenate((plan_dict['action_trajectory'], np.array(plan.gripper_action_trajectory).astype(np.float32)), axis=1) + # plan_dict['action_period'] = plan.action_period +- self.plan_queue.add(plan.header, plan.obs_header.stamp, plan_dataclass) ++ self.plan_queue.add(plan.header, plan.obs_anchor_header.stamp, plan_dataclass) + else: + rospy.logwarn('In process of resetting, ignoring plan') + pass diff --git a/210407/wandb/run-20241210_210414-ob7chcle/files/media/videos/eval/0_eval_0_242331cb3a64f66021df.mp4 b/210407/wandb/run-20241210_210414-ob7chcle/files/media/videos/eval/0_eval_0_242331cb3a64f66021df.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5ae81c700a986d6390e22d5c7f44c214169b40e5 --- /dev/null +++ b/210407/wandb/run-20241210_210414-ob7chcle/files/media/videos/eval/0_eval_0_242331cb3a64f66021df.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:242331cb3a64f66021dfad95eeae9d0d8610c587d62363724138c291f7e496ce +size 1091529 diff --git a/210407/wandb/run-20241210_210414-ob7chcle/files/media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 b/210407/wandb/run-20241210_210414-ob7chcle/files/media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa868fbb75c51fb59ae42ff30716054804074343 Binary files /dev/null and b/210407/wandb/run-20241210_210414-ob7chcle/files/media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 differ diff --git a/210407/wandb/run-20241210_210414-ob7chcle/files/output.log b/210407/wandb/run-20241210_210414-ob7chcle/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..0cc26e4de51e6427dd0ac10d3897025b23a5be49 --- /dev/null +++ b/210407/wandb/run-20241210_210414-ob7chcle/files/output.log @@ -0,0 +1,1285 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1733882708.019009]: resetting environment +[INFO] [1733882708.023743]: cleared current plan +[INFO] [1733882708.024913]: moving to home +[INFO] [1733882709.729667]: reached home +[INFO] [1733882709.731097]: reset action history +[INFO] [1733882712.439471]: environment reset +Starting episode 0 +[INFO] [1733882712.442006]: resetting environment +[INFO] [1733882712.445620]: cleared current plan +[INFO] [1733882712.446719]: moving to home +[INFO] [1733882713.448835]: reached home +[INFO] [1733882713.450003]: reset action history +[INFO] [1733882716.159782]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1733882717.586598]: resetting environment +[INFO] [1733882717.593480]: cleared current plan +[INFO] [1733882717.594324]: moving to home +[INFO] [1733882718.596403]: reached home +[INFO] [1733882718.597975]: reset action history +[INFO] [1733882721.306381]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/episode_rosbags/episode_0_2024-12-10-21-05-21.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-599894.log +started roslaunch server http://158.130.50.37:44529/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [600194] +process[print_text-2]: started with pid [600218] +process[rosbag_record-3]: started with pid [600219] +started bagging! +[INFO] [1733882722.053888]: received mask_obs with header seq: 114 +stamp: + secs: 1733882722 + nsecs: 48879146 +frame_id: '' and grasped_object_mask header seq: 31506 +stamp: + secs: 1733882721 + nsecs: 834905148 +frame_id: "camera_color_optical_frame" +[INFO] [1733882722.242116]: received mask_obs with header seq: 115 +stamp: + secs: 1733882722 + nsecs: 242531061 +frame_id: '' and grasped_object_mask header seq: 31512 +stamp: + secs: 1733882722 + nsecs: 34748554 +frame_id: "camera_color_optical_frame" +[INFO] [1733882722.438313]: received mask_obs with header seq: 116 +stamp: + secs: 1733882722 + nsecs: 438178777 +frame_id: '' and grasped_object_mask header seq: 31518 +stamp: + secs: 1733882722 + nsecs: 234591007 +frame_id: "camera_color_optical_frame" +[INFO] [1733882722.642956]: received mask_obs with header seq: 117 +stamp: + secs: 1733882722 + nsecs: 643582820 +frame_id: '' and grasped_object_mask header seq: 31524 +stamp: + secs: 1733882722 + nsecs: 434433460 +frame_id: "camera_color_optical_frame" +[INFO] [1733882722.855864]: received mask_obs with header seq: 118 +stamp: + secs: 1733882722 + nsecs: 856016874 +frame_id: '' and grasped_object_mask header seq: 31530 +stamp: + secs: 1733882722 + nsecs: 634273052 +frame_id: "camera_color_optical_frame" +[INFO] [1733882723.043437]: received mask_obs with header seq: 119 +stamp: + secs: 1733882723 + nsecs: 44147491 +frame_id: '' and grasped_object_mask header seq: 31536 +stamp: + secs: 1733882722 + nsecs: 834112883 +frame_id: "camera_color_optical_frame" +[INFO] [1733882723.255068]: received mask_obs with header seq: 120 +stamp: + secs: 1733882723 + nsecs: 255773782 +frame_id: '' and grasped_object_mask header seq: 31542 +stamp: + secs: 1733882723 + nsecs: 33952713 +frame_id: "camera_color_optical_frame" +[INFO] [1733882723.457205]: received mask_obs with header seq: 121 +stamp: + secs: 1733882723 + nsecs: 457902193 +frame_id: '' and grasped_object_mask header seq: 31548 +stamp: + secs: 1733882723 + nsecs: 233793497 +frame_id: "camera_color_optical_frame" +[INFO] [1733882723.672633]: received mask_obs with header seq: 122 +stamp: + secs: 1733882723 + nsecs: 673328876 +frame_id: '' and grasped_object_mask header seq: 31554 +stamp: + secs: 1733882723 + nsecs: 433632374 +frame_id: "camera_color_optical_frame" +[INFO] [1733882723.868197]: received mask_obs with header seq: 123 +stamp: + secs: 1733882723 + nsecs: 868912935 +frame_id: '' and grasped_object_mask header seq: 31560 +stamp: + secs: 1733882723 + nsecs: 633465290 +frame_id: "camera_color_optical_frame" +[INFO] [1733882724.063814]: received mask_obs with header seq: 124 +stamp: + secs: 1733882724 + nsecs: 64529418 +frame_id: '' and grasped_object_mask header seq: 31566 +stamp: + secs: 1733882723 + nsecs: 833298206 +frame_id: "camera_color_optical_frame" +[INFO] [1733882724.270881]: received mask_obs with header seq: 125 +stamp: + secs: 1733882724 + nsecs: 271565675 +frame_id: '' and grasped_object_mask header seq: 31572 +stamp: + secs: 1733882724 + nsecs: 33130884 +frame_id: "camera_color_optical_frame" +[INFO] [1733882724.456985]: received mask_obs with header seq: 126 +stamp: + secs: 1733882724 + nsecs: 457637548 +frame_id: '' and grasped_object_mask header seq: 31578 +stamp: + secs: 1733882724 + nsecs: 232963324 +frame_id: "camera_color_optical_frame" +[INFO] [1733882724.651581]: received mask_obs with header seq: 127 +stamp: + secs: 1733882724 + nsecs: 652227878 +frame_id: '' and grasped_object_mask header seq: 31584 +stamp: + secs: 1733882724 + nsecs: 432796717 +frame_id: "camera_color_optical_frame" +[INFO] [1733882724.870804]: received mask_obs with header seq: 128 +stamp: + secs: 1733882724 + nsecs: 871049642 +frame_id: '' and grasped_object_mask header seq: 31590 +stamp: + secs: 1733882724 + nsecs: 632633924 +frame_id: "camera_color_optical_frame" +[INFO] [1733882725.075478]: received mask_obs with header seq: 129 +stamp: + secs: 1733882725 + nsecs: 76172590 +frame_id: '' and grasped_object_mask header seq: 31596 +stamp: + secs: 1733882724 + nsecs: 832470655 +frame_id: "camera_color_optical_frame" +[INFO] [1733882725.273276]: received mask_obs with header seq: 130 +stamp: + secs: 1733882725 + nsecs: 273989677 +frame_id: '' and grasped_object_mask header seq: 31602 +stamp: + secs: 1733882725 + nsecs: 32306910 +frame_id: "camera_color_optical_frame" +[INFO] [1733882725.473219]: received mask_obs with header seq: 131 +stamp: + secs: 1733882725 + nsecs: 473895311 +frame_id: '' and grasped_object_mask header seq: 31608 +stamp: + secs: 1733882725 + nsecs: 232143402 +frame_id: "camera_color_optical_frame" +[INFO] [1733882725.658554]: received mask_obs with header seq: 132 +stamp: + secs: 1733882725 + nsecs: 659076929 +frame_id: '' and grasped_object_mask header seq: 31614 +stamp: + secs: 1733882725 + nsecs: 431979418 +frame_id: "camera_color_optical_frame" +[INFO] [1733882725.861954]: received mask_obs with header seq: 133 +stamp: + secs: 1733882725 + nsecs: 862345933 +frame_id: '' and grasped_object_mask header seq: 31620 +stamp: + secs: 1733882725 + nsecs: 631814957 +frame_id: "camera_color_optical_frame" +[INFO] [1733882726.058597]: received mask_obs with header seq: 134 +stamp: + secs: 1733882726 + nsecs: 58791875 +frame_id: '' and grasped_object_mask header seq: 31626 +stamp: + secs: 1733882725 + nsecs: 831651449 +frame_id: "camera_color_optical_frame" +[INFO] [1733882726.259522]: received mask_obs with header seq: 135 +stamp: + secs: 1733882726 + nsecs: 259843826 +frame_id: '' and grasped_object_mask header seq: 31632 +stamp: + secs: 1733882726 + nsecs: 31486511 +frame_id: "camera_color_optical_frame" +[INFO] [1733882726.464562]: received mask_obs with header seq: 136 +stamp: + secs: 1733882726 + nsecs: 465265750 +frame_id: '' and grasped_object_mask header seq: 31638 +stamp: + secs: 1733882726 + nsecs: 231322050 +frame_id: "camera_color_optical_frame" +[INFO] [1733882726.657005]: received mask_obs with header seq: 137 +stamp: + secs: 1733882726 + nsecs: 657728195 +frame_id: '' and grasped_object_mask header seq: 31644 +stamp: + secs: 1733882726 + nsecs: 431157827 +frame_id: "camera_color_optical_frame" +[INFO] [1733882726.853767]: received mask_obs with header seq: 138 +stamp: + secs: 1733882726 + nsecs: 853689193 +frame_id: '' and grasped_object_mask header seq: 31650 +stamp: + secs: 1733882726 + nsecs: 630995035 +frame_id: "camera_color_optical_frame" +[INFO] [1733882727.054479]: received mask_obs with header seq: 139 +stamp: + secs: 1733882727 + nsecs: 54865360 +frame_id: '' and grasped_object_mask header seq: 31656 +stamp: + secs: 1733882726 + nsecs: 830832481 +frame_id: "camera_color_optical_frame" +[INFO] [1733882727.252600]: received mask_obs with header seq: 140 +stamp: + secs: 1733882727 + nsecs: 252785444 +frame_id: '' and grasped_object_mask header seq: 31662 +stamp: + secs: 1733882727 + nsecs: 30670881 +frame_id: "camera_color_optical_frame" +[INFO] [1733882727.462991]: received mask_obs with header seq: 141 +stamp: + secs: 1733882727 + nsecs: 463722467 +frame_id: '' and grasped_object_mask header seq: 31668 +stamp: + secs: 1733882727 + nsecs: 230508089 +frame_id: "camera_color_optical_frame" +[INFO] [1733882727.659335]: received mask_obs with header seq: 142 +stamp: + secs: 1733882727 + nsecs: 660079240 +frame_id: '' and grasped_object_mask header seq: 31674 +stamp: + secs: 1733882727 + nsecs: 430346251 +frame_id: "camera_color_optical_frame" +[INFO] [1733882727.878530]: received mask_obs with header seq: 143 +stamp: + secs: 1733882727 + nsecs: 878180265 +frame_id: '' and grasped_object_mask header seq: 31680 +stamp: + secs: 1733882727 + nsecs: 630188227 +frame_id: "camera_color_optical_frame" +[INFO] [1733882728.099251]: received mask_obs with header seq: 144 +stamp: + secs: 1733882728 + nsecs: 99953413 +frame_id: '' and grasped_object_mask header seq: 31687 +stamp: + secs: 1733882727 + nsecs: 863338232 +frame_id: "camera_color_optical_frame" +[INFO] [1733882728.290529]: received mask_obs with header seq: 145 +stamp: + secs: 1733882728 + nsecs: 290887117 +frame_id: '' and grasped_object_mask header seq: 31693 +stamp: + secs: 1733882728 + nsecs: 63180685 +frame_id: "camera_color_optical_frame" +[INFO] [1733882728.493851]: received mask_obs with header seq: 146 +stamp: + secs: 1733882728 + nsecs: 494552135 +frame_id: '' and grasped_object_mask header seq: 31699 +stamp: + secs: 1733882728 + nsecs: 263023138 +frame_id: "camera_color_optical_frame" +[INFO] [1733882728.705236]: received mask_obs with header seq: 147 +stamp: + secs: 1733882728 + nsecs: 705916881 +frame_id: '' and grasped_object_mask header seq: 31705 +stamp: + secs: 1733882728 + nsecs: 462864399 +frame_id: "camera_color_optical_frame" +[INFO] [1733882728.907349]: received mask_obs with header seq: 148 +stamp: + secs: 1733882728 + nsecs: 907902240 +frame_id: '' and grasped_object_mask header seq: 31711 +stamp: + secs: 1733882728 + nsecs: 662702799 +frame_id: "camera_color_optical_frame" +[INFO] [1733882729.123544]: received mask_obs with header seq: 149 +stamp: + secs: 1733882729 + nsecs: 124239444 +frame_id: '' and grasped_object_mask header seq: 31718 +stamp: + secs: 1733882728 + nsecs: 895848036 +frame_id: "camera_color_optical_frame" +[INFO] [1733882729.338554]: received mask_obs with header seq: 150 +stamp: + secs: 1733882729 + nsecs: 339276552 +frame_id: '' and grasped_object_mask header seq: 31724 +stamp: + secs: 1733882729 + nsecs: 95686436 +frame_id: "camera_color_optical_frame" +[INFO] [1733882729.520767]: received mask_obs with header seq: 151 +stamp: + secs: 1733882729 + nsecs: 520812273 +frame_id: '' and grasped_object_mask header seq: 31730 +stamp: + secs: 1733882729 + nsecs: 295524836 +frame_id: "camera_color_optical_frame" +[INFO] [1733882729.713979]: received mask_obs with header seq: 152 +stamp: + secs: 1733882729 + nsecs: 714640617 +frame_id: '' and grasped_object_mask header seq: 31736 +stamp: + secs: 1733882729 + nsecs: 495365620 +frame_id: "camera_color_optical_frame" +[INFO] [1733882729.913416]: received mask_obs with header seq: 153 +stamp: + secs: 1733882729 + nsecs: 913074016 +frame_id: '' and grasped_object_mask header seq: 31741 +stamp: + secs: 1733882729 + nsecs: 661899090 +frame_id: "camera_color_optical_frame" +[INFO] [1733882730.108793]: received mask_obs with header seq: 154 +stamp: + secs: 1733882730 + nsecs: 109471797 +frame_id: '' and grasped_object_mask header seq: 31747 +stamp: + secs: 1733882729 + nsecs: 861740589 +frame_id: "camera_color_optical_frame" +[INFO] [1733882730.302778]: received mask_obs with header seq: 155 +stamp: + secs: 1733882730 + nsecs: 303365468 +frame_id: '' and grasped_object_mask header seq: 31753 +stamp: + secs: 1733882730 + nsecs: 61582804 +frame_id: "camera_color_optical_frame" +[INFO] [1733882730.489773]: received mask_obs with header seq: 156 +stamp: + secs: 1733882730 + nsecs: 490137338 +frame_id: '' and grasped_object_mask header seq: 31759 +stamp: + secs: 1733882730 + nsecs: 261424303 +frame_id: "camera_color_optical_frame" +[INFO] [1733882730.688014]: received mask_obs with header seq: 157 +stamp: + secs: 1733882730 + nsecs: 688325881 +frame_id: '' and grasped_object_mask header seq: 31765 +stamp: + secs: 1733882730 + nsecs: 461265564 +frame_id: "camera_color_optical_frame" +[INFO] [1733882730.890480]: received mask_obs with header seq: 158 +stamp: + secs: 1733882730 + nsecs: 890745401 +frame_id: '' and grasped_object_mask header seq: 31771 +stamp: + secs: 1733882730 + nsecs: 661106586 +frame_id: "camera_color_optical_frame" +[INFO] [1733882731.091830]: received mask_obs with header seq: 159 +stamp: + secs: 1733882731 + nsecs: 92510223 +frame_id: '' and grasped_object_mask header seq: 31777 +stamp: + secs: 1733882730 + nsecs: 860948324 +frame_id: "camera_color_optical_frame" +[INFO] [1733882731.285396]: received mask_obs with header seq: 160 +stamp: + secs: 1733882731 + nsecs: 286172151 +frame_id: '' and grasped_object_mask header seq: 31783 +stamp: + secs: 1733882731 + nsecs: 60789824 +frame_id: "camera_color_optical_frame" +[INFO] [1733882731.483410]: received mask_obs with header seq: 161 +stamp: + secs: 1733882731 + nsecs: 484132289 +frame_id: '' and grasped_object_mask header seq: 31789 +stamp: + secs: 1733882731 + nsecs: 260632753 +frame_id: "camera_color_optical_frame" +[INFO] [1733882731.681967]: received mask_obs with header seq: 162 +stamp: + secs: 1733882731 + nsecs: 682367324 +frame_id: '' and grasped_object_mask header seq: 31795 +stamp: + secs: 1733882731 + nsecs: 460474491 +frame_id: "camera_color_optical_frame" +[INFO] [1733882731.881424]: received mask_obs with header seq: 163 +stamp: + secs: 1733882731 + nsecs: 882004022 +frame_id: '' and grasped_object_mask header seq: 31801 +stamp: + secs: 1733882731 + nsecs: 660314560 +frame_id: "camera_color_optical_frame" +[INFO] [1733882732.080389]: received mask_obs with header seq: 164 +stamp: + secs: 1733882732 + nsecs: 80821037 +frame_id: '' and grasped_object_mask header seq: 31807 +stamp: + secs: 1733882731 + nsecs: 860154390 +frame_id: "camera_color_optical_frame" +[INFO] [1733882732.276461]: received mask_obs with header seq: 165 +stamp: + secs: 1733882732 + nsecs: 277194499 +frame_id: '' and grasped_object_mask header seq: 31813 +stamp: + secs: 1733882732 + nsecs: 59993982 +frame_id: "camera_color_optical_frame" +[INFO] [1733882732.475675]: received mask_obs with header seq: 166 +stamp: + secs: 1733882732 + nsecs: 475312232 +frame_id: '' and grasped_object_mask header seq: 31819 +stamp: + secs: 1733882732 + nsecs: 259833813 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -224012663 for nearest: 1733882732235661457 - target: 1733882732459674120 at idx 157 +[INFO] [1733882732.668931]: received mask_obs with header seq: 167 +stamp: + secs: 1733882732 + nsecs: 669728040 +frame_id: '' and grasped_object_mask header seq: 31824 +stamp: + secs: 1733882732 + nsecs: 426367521 +frame_id: "camera_color_optical_frame" +[INFO] [1733882732.859527]: received mask_obs with header seq: 168 +stamp: + secs: 1733882732 + nsecs: 860039234 +frame_id: '' and grasped_object_mask header seq: 31830 +stamp: + secs: 1733882732 + nsecs: 626210690 +frame_id: "camera_color_optical_frame" +[INFO] [1733882733.058874]: received mask_obs with header seq: 169 +stamp: + secs: 1733882733 + nsecs: 59504032 +frame_id: '' and grasped_object_mask header seq: 31836 +stamp: + secs: 1733882732 + nsecs: 826054335 +frame_id: "camera_color_optical_frame" +[INFO] [1733882733.279345]: received mask_obs with header seq: 170 +stamp: + secs: 1733882733 + nsecs: 279613733 +frame_id: '' and grasped_object_mask header seq: 31842 +stamp: + secs: 1733882733 + nsecs: 25898457 +frame_id: "camera_color_optical_frame" +[INFO] [1733882733.474381]: received mask_obs with header seq: 171 +stamp: + secs: 1733882733 + nsecs: 474082469 +frame_id: '' and grasped_object_mask header seq: 31849 +stamp: + secs: 1733882733 + nsecs: 259050369 +frame_id: "camera_color_optical_frame" +[INFO] [1733882733.681119]: received mask_obs with header seq: 172 +stamp: + secs: 1733882733 + nsecs: 681621789 +frame_id: '' and grasped_object_mask header seq: 31855 +stamp: + secs: 1733882733 + nsecs: 458893776 +frame_id: "camera_color_optical_frame" +[INFO] [1733882733.879652]: received mask_obs with header seq: 173 +stamp: + secs: 1733882733 + nsecs: 880379438 +frame_id: '' and grasped_object_mask header seq: 31861 +stamp: + secs: 1733882733 + nsecs: 658732653 +frame_id: "camera_color_optical_frame" +[INFO] [1733882734.076964]: received mask_obs with header seq: 174 +stamp: + secs: 1733882734 + nsecs: 77689647 +frame_id: '' and grasped_object_mask header seq: 31866 +stamp: + secs: 1733882733 + nsecs: 825265646 +frame_id: "camera_color_optical_frame" +[INFO] [1733882734.274835]: received mask_obs with header seq: 175 +stamp: + secs: 1733882734 + nsecs: 275566339 +frame_id: '' and grasped_object_mask header seq: 31873 +stamp: + secs: 1733882734 + nsecs: 58411598 +frame_id: "camera_color_optical_frame" +[INFO] [1733882734.475413]: received mask_obs with header seq: 176 +stamp: + secs: 1733882734 + nsecs: 475030899 +frame_id: '' and grasped_object_mask header seq: 31879 +stamp: + secs: 1733882734 + nsecs: 258250475 +frame_id: "camera_color_optical_frame" +[INFO] [1733882734.684631]: received mask_obs with header seq: 177 +stamp: + secs: 1733882734 + nsecs: 684263467 +frame_id: '' and grasped_object_mask header seq: 31885 +stamp: + secs: 1733882734 + nsecs: 458089352 +frame_id: "camera_color_optical_frame" +[INFO] [1733882734.881454]: received mask_obs with header seq: 178 +stamp: + secs: 1733882734 + nsecs: 882217168 +frame_id: '' and grasped_object_mask header seq: 31891 +stamp: + secs: 1733882734 + nsecs: 657929182 +frame_id: "camera_color_optical_frame" +[INFO] [1733882735.078412]: received mask_obs with header seq: 179 +stamp: + secs: 1733882735 + nsecs: 78201293 +frame_id: '' and grasped_object_mask header seq: 31897 +stamp: + secs: 1733882734 + nsecs: 857769251 +frame_id: "camera_color_optical_frame" +[INFO] [1733882735.276782]: received mask_obs with header seq: 180 +stamp: + secs: 1733882735 + nsecs: 277486085 +frame_id: '' and grasped_object_mask header seq: 31903 +stamp: + secs: 1733882735 + nsecs: 57610035 +frame_id: "camera_color_optical_frame" +[INFO] [1733882735.473451]: received mask_obs with header seq: 181 +stamp: + secs: 1733882735 + nsecs: 473750829 +frame_id: '' and grasped_object_mask header seq: 31909 +stamp: + secs: 1733882735 + nsecs: 257450104 +frame_id: "camera_color_optical_frame" +[INFO] [1733882735.673778]: received mask_obs with header seq: 182 +stamp: + secs: 1733882735 + nsecs: 672456741 +frame_id: '' and grasped_object_mask header seq: 31915 +stamp: + secs: 1733882735 + nsecs: 457290173 +frame_id: "camera_color_optical_frame" +[INFO] [1733882735.878548]: received mask_obs with header seq: 183 +stamp: + secs: 1733882735 + nsecs: 879298925 +frame_id: '' and grasped_object_mask header seq: 31921 +stamp: + secs: 1733882735 + nsecs: 657128334 +frame_id: "camera_color_optical_frame" +[INFO] [1733882736.076265]: received mask_obs with header seq: 184 +stamp: + secs: 1733882736 + nsecs: 77043294 +frame_id: '' and grasped_object_mask header seq: 31927 +stamp: + secs: 1733882735 + nsecs: 856966734 +frame_id: "camera_color_optical_frame" +[INFO] [1733882736.278515]: received mask_obs with header seq: 185 +stamp: + secs: 1733882736 + nsecs: 277301549 +frame_id: '' and grasped_object_mask header seq: 31933 +stamp: + secs: 1733882736 + nsecs: 56804895 +frame_id: "camera_color_optical_frame" +[INFO] [1733882736.471332]: received mask_obs with header seq: 186 +stamp: + secs: 1733882736 + nsecs: 471755981 +frame_id: '' and grasped_object_mask header seq: 31939 +stamp: + secs: 1733882736 + nsecs: 256644011 +frame_id: "camera_color_optical_frame" +[INFO] [1733882736.664083]: received mask_obs with header seq: 187 +stamp: + secs: 1733882736 + nsecs: 664746999 +frame_id: '' and grasped_object_mask header seq: 31944 +stamp: + secs: 1733882736 + nsecs: 423175335 +frame_id: "camera_color_optical_frame" +[INFO] [1733882736.856564]: received mask_obs with header seq: 188 +stamp: + secs: 1733882736 + nsecs: 857145786 +frame_id: '' and grasped_object_mask header seq: 31950 +stamp: + secs: 1733882736 + nsecs: 623027086 +frame_id: "camera_color_optical_frame" +[INFO] [1733882737.054379]: received mask_obs with header seq: 189 +stamp: + secs: 1733882737 + nsecs: 54893493 +frame_id: '' and grasped_object_mask header seq: 31956 +stamp: + secs: 1733882736 + nsecs: 822880268 +frame_id: "camera_color_optical_frame" +[INFO] [1733882737.253503]: received mask_obs with header seq: 190 +stamp: + secs: 1733882737 + nsecs: 254244327 +frame_id: '' and grasped_object_mask header seq: 31962 +stamp: + secs: 1733882737 + nsecs: 22733212 +frame_id: "camera_color_optical_frame" +[INFO] [1733882737.452823]: received mask_obs with header seq: 191 +stamp: + secs: 1733882737 + nsecs: 453443765 +frame_id: '' and grasped_object_mask header seq: 31968 +stamp: + secs: 1733882737 + nsecs: 222586632 +frame_id: "camera_color_optical_frame" +[INFO] [1733882737.657336]: received mask_obs with header seq: 192 +stamp: + secs: 1733882737 + nsecs: 657974243 +frame_id: '' and grasped_object_mask header seq: 31974 +stamp: + secs: 1733882737 + nsecs: 422440529 +frame_id: "camera_color_optical_frame" +[INFO] [1733882737.851166]: received mask_obs with header seq: 193 +stamp: + secs: 1733882737 + nsecs: 851891756 +frame_id: '' and grasped_object_mask header seq: 31980 +stamp: + secs: 1733882737 + nsecs: 622284889 +frame_id: "camera_color_optical_frame" +[INFO] [1733882738.050410]: received mask_obs with header seq: 194 +stamp: + secs: 1733882738 + nsecs: 51139116 +frame_id: '' and grasped_object_mask header seq: 31986 +stamp: + secs: 1733882737 + nsecs: 822129726 +frame_id: "camera_color_optical_frame" +[INFO] [1733882738.252615]: received mask_obs with header seq: 195 +stamp: + secs: 1733882738 + nsecs: 253381729 +frame_id: '' and grasped_object_mask header seq: 31992 +stamp: + secs: 1733882738 + nsecs: 21975756 +frame_id: "camera_color_optical_frame" +[INFO] [1733882738.461679]: received mask_obs with header seq: 196 +stamp: + secs: 1733882738 + nsecs: 462370157 +frame_id: '' and grasped_object_mask header seq: 31998 +stamp: + secs: 1733882738 + nsecs: 221821308 +frame_id: "camera_color_optical_frame" +[INFO] [1733882738.654291]: received mask_obs with header seq: 197 +stamp: + secs: 1733882738 + nsecs: 654976367 +frame_id: '' and grasped_object_mask header seq: 32004 +stamp: + secs: 1733882738 + nsecs: 421667337 +frame_id: "camera_color_optical_frame" +[INFO] [1733882738.850199]: received mask_obs with header seq: 198 +stamp: + secs: 1733882738 + nsecs: 850958347 +frame_id: '' and grasped_object_mask header seq: 32010 +stamp: + secs: 1733882738 + nsecs: 621500492 +frame_id: "camera_color_optical_frame" +[INFO] [1733882739.049589]: received mask_obs with header seq: 199 +stamp: + secs: 1733882739 + nsecs: 49774408 +frame_id: '' and grasped_object_mask header seq: 32016 +stamp: + secs: 1733882738 + nsecs: 821333647 +frame_id: "camera_color_optical_frame" +[INFO] [1733882739.252296]: received mask_obs with header seq: 200 +stamp: + secs: 1733882739 + nsecs: 252633094 +frame_id: '' and grasped_object_mask header seq: 32022 +stamp: + secs: 1733882739 + nsecs: 21167040 +frame_id: "camera_color_optical_frame" +[INFO] [1733882739.456659]: received mask_obs with header seq: 201 +stamp: + secs: 1733882739 + nsecs: 457164287 +frame_id: '' and grasped_object_mask header seq: 32028 +stamp: + secs: 1733882739 + nsecs: 220999718 +frame_id: "camera_color_optical_frame" +[INFO] [1733882739.656339]: received mask_obs with header seq: 202 +stamp: + secs: 1733882739 + nsecs: 657097578 +frame_id: '' and grasped_object_mask header seq: 32034 +stamp: + secs: 1733882739 + nsecs: 420833826 +frame_id: "camera_color_optical_frame" +[INFO] [1733882739.855413]: received mask_obs with header seq: 203 +stamp: + secs: 1733882739 + nsecs: 856137990 +frame_id: '' and grasped_object_mask header seq: 32040 +stamp: + secs: 1733882739 + nsecs: 620669603 +frame_id: "camera_color_optical_frame" +[INFO] [1733882740.050060]: received mask_obs with header seq: 204 +stamp: + secs: 1733882740 + nsecs: 50755262 +frame_id: '' and grasped_object_mask header seq: 32046 +stamp: + secs: 1733882739 + nsecs: 820505619 +frame_id: "camera_color_optical_frame" +[INFO] [1733882740.255058]: received mask_obs with header seq: 205 +stamp: + secs: 1733882740 + nsecs: 255806922 +frame_id: '' and grasped_object_mask header seq: 32052 +stamp: + secs: 1733882740 + nsecs: 20341158 +frame_id: "camera_color_optical_frame" +[INFO] [1733882740.463739]: received mask_obs with header seq: 206 +stamp: + secs: 1733882740 + nsecs: 464346647 +frame_id: '' and grasped_object_mask header seq: 32058 +stamp: + secs: 1733882740 + nsecs: 220176458 +frame_id: "camera_color_optical_frame" +[INFO] [1733882740.659454]: received mask_obs with header seq: 207 +stamp: + secs: 1733882740 + nsecs: 659945487 +frame_id: '' and grasped_object_mask header seq: 32064 +stamp: + secs: 1733882740 + nsecs: 420011520 +frame_id: "camera_color_optical_frame" +[INFO] [1733882740.853475]: received mask_obs with header seq: 208 +stamp: + secs: 1733882740 + nsecs: 854121923 +frame_id: '' and grasped_object_mask header seq: 32070 +stamp: + secs: 1733882740 + nsecs: 619850636 +frame_id: "camera_color_optical_frame" +[INFO] [1733882741.054072]: received mask_obs with header seq: 209 +stamp: + secs: 1733882741 + nsecs: 54691076 +frame_id: '' and grasped_object_mask header seq: 32076 +stamp: + secs: 1733882740 + nsecs: 819688559 +frame_id: "camera_color_optical_frame" +[INFO] [1733882741.252034]: received mask_obs with header seq: 210 +stamp: + secs: 1733882741 + nsecs: 252728223 +frame_id: '' and grasped_object_mask header seq: 32082 +stamp: + secs: 1733882741 + nsecs: 19526482 +frame_id: "camera_color_optical_frame" +[INFO] [1733882741.453009]: received mask_obs with header seq: 211 +stamp: + secs: 1733882741 + nsecs: 453506231 +frame_id: '' and grasped_object_mask header seq: 32088 +stamp: + secs: 1733882741 + nsecs: 219364166 +frame_id: "camera_color_optical_frame" +[INFO] [1733882741.652107]: received mask_obs with header seq: 212 +stamp: + secs: 1733882741 + nsecs: 652641057 +frame_id: '' and grasped_object_mask header seq: 32094 +stamp: + secs: 1733882741 + nsecs: 419201851 +frame_id: "camera_color_optical_frame" +[INFO] [1733882741.849829]: received mask_obs with header seq: 213 +stamp: + secs: 1733882741 + nsecs: 850602865 +frame_id: '' and grasped_object_mask header seq: 32100 +stamp: + secs: 1733882741 + nsecs: 619036913 +frame_id: "camera_color_optical_frame" +[INFO] [1733882742.050523]: received mask_obs with header seq: 214 +stamp: + secs: 1733882742 + nsecs: 51265954 +frame_id: '' and grasped_object_mask header seq: 32106 +stamp: + secs: 1733882741 + nsecs: 818872213 +frame_id: "camera_color_optical_frame" +[INFO] [1733882742.251031]: received mask_obs with header seq: 215 +stamp: + secs: 1733882742 + nsecs: 251769065 +frame_id: '' and grasped_object_mask header seq: 32112 +stamp: + secs: 1733882742 + nsecs: 18706799 +frame_id: "camera_color_optical_frame" +[INFO] [1733882742.452404]: received mask_obs with header seq: 216 +stamp: + secs: 1733882742 + nsecs: 453162908 +frame_id: '' and grasped_object_mask header seq: 32118 +stamp: + secs: 1733882742 + nsecs: 218540668 +frame_id: "camera_color_optical_frame" +[INFO] [1733882742.659943]: received mask_obs with header seq: 217 +stamp: + secs: 1733882742 + nsecs: 660651683 +frame_id: '' and grasped_object_mask header seq: 32124 +stamp: + secs: 1733882742 + nsecs: 418374538 +frame_id: "camera_color_optical_frame" +[INFO] [1733882742.851243]: received mask_obs with header seq: 218 +stamp: + secs: 1733882742 + nsecs: 851963758 +frame_id: '' and grasped_object_mask header seq: 32130 +stamp: + secs: 1733882742 + nsecs: 618206978 +frame_id: "camera_color_optical_frame" +[INFO] [1733882743.053159]: received mask_obs with header seq: 219 +stamp: + secs: 1733882743 + nsecs: 53831577 +frame_id: '' and grasped_object_mask header seq: 32136 +stamp: + secs: 1733882742 + nsecs: 818039417 +frame_id: "camera_color_optical_frame" +[INFO] [1733882743.253214]: received mask_obs with header seq: 220 +stamp: + secs: 1733882743 + nsecs: 253848552 +frame_id: '' and grasped_object_mask header seq: 32142 +stamp: + secs: 1733882743 + nsecs: 17872334 +frame_id: "camera_color_optical_frame" +[INFO] [1733882743.471638]: received mask_obs with header seq: 221 +stamp: + secs: 1733882743 + nsecs: 472417116 +frame_id: '' and grasped_object_mask header seq: 32148 +stamp: + secs: 1733882743 + nsecs: 217703819 +frame_id: "camera_color_optical_frame" +[INFO] [1733882743.671971]: received mask_obs with header seq: 222 +stamp: + secs: 1733882743 + nsecs: 672483921 +frame_id: '' and grasped_object_mask header seq: 32155 +stamp: + secs: 1733882743 + nsecs: 450840950 +frame_id: "camera_color_optical_frame" +[INFO] [1733882743.857297]: received mask_obs with header seq: 223 +stamp: + secs: 1733882743 + nsecs: 857908964 +frame_id: '' and grasped_object_mask header seq: 32160 +stamp: + secs: 1733882743 + nsecs: 617372990 +frame_id: "camera_color_optical_frame" +[INFO] [1733882744.051181]: received mask_obs with header seq: 224 +stamp: + secs: 1733882744 + nsecs: 51657915 +frame_id: '' and grasped_object_mask header seq: 32166 +stamp: + secs: 1733882743 + nsecs: 817210913 +frame_id: "camera_color_optical_frame" +[INFO] [1733882744.250471]: received mask_obs with header seq: 225 +stamp: + secs: 1733882744 + nsecs: 251115560 +frame_id: '' and grasped_object_mask header seq: 32172 +stamp: + secs: 1733882744 + nsecs: 17048597 +frame_id: "camera_color_optical_frame" +[INFO] [1733882744.453214]: received mask_obs with header seq: 226 +stamp: + secs: 1733882744 + nsecs: 453812599 +frame_id: '' and grasped_object_mask header seq: 32178 +stamp: + secs: 1733882744 + nsecs: 216887474 +frame_id: "camera_color_optical_frame" +[INFO] [1733882744.669318]: received mask_obs with header seq: 227 +stamp: + secs: 1733882744 + nsecs: 670104026 +frame_id: '' and grasped_object_mask header seq: 32184 +stamp: + secs: 1733882744 + nsecs: 416725397 +frame_id: "camera_color_optical_frame" +[INFO] [1733882744.878128]: received mask_obs with header seq: 228 +stamp: + secs: 1733882744 + nsecs: 878016233 +frame_id: '' and grasped_object_mask header seq: 32191 +stamp: + secs: 1733882744 + nsecs: 649865627 +frame_id: "camera_color_optical_frame" +[INFO] [1733882745.086062]: received mask_obs with header seq: 229 +stamp: + secs: 1733882745 + nsecs: 86757659 +frame_id: '' and grasped_object_mask header seq: 32197 +stamp: + secs: 1733882744 + nsecs: 849701643 +frame_id: "camera_color_optical_frame" +[INFO] [1733882745.280597]: received mask_obs with header seq: 230 +stamp: + secs: 1733882745 + nsecs: 281128883 +frame_id: '' and grasped_object_mask header seq: 32203 +stamp: + secs: 1733882745 + nsecs: 49536467 +frame_id: "camera_color_optical_frame" +[INFO] [1733882745.479238]: received mask_obs with header seq: 231 +stamp: + secs: 1733882745 + nsecs: 479716539 +frame_id: '' and grasped_object_mask header seq: 32209 +stamp: + secs: 1733882745 + nsecs: 249371052 +frame_id: "camera_color_optical_frame" +[INFO] [1733882745.686511]: received mask_obs with header seq: 232 +stamp: + secs: 1733882745 + nsecs: 687139987 +frame_id: '' and grasped_object_mask header seq: 32215 +stamp: + secs: 1733882745 + nsecs: 449206352 +frame_id: "camera_color_optical_frame" +[INFO] [1733882745.869770]: received mask_obs with header seq: 233 +stamp: + secs: 1733882745 + nsecs: 870471954 +frame_id: '' and grasped_object_mask header seq: 32221 +stamp: + secs: 1733882745 + nsecs: 649044991 +frame_id: "camera_color_optical_frame" +[INFO] [1733882746.052903]: received mask_obs with header seq: 234 +stamp: + secs: 1733882746 + nsecs: 53678035 +frame_id: '' and grasped_object_mask header seq: 32226 +stamp: + secs: 1733882745 + nsecs: 815577984 +frame_id: "camera_color_optical_frame" +[INFO] [1733882746.251527]: received mask_obs with header seq: 235 +stamp: + secs: 1733882746 + nsecs: 252217292 +frame_id: '' and grasped_object_mask header seq: 32232 +stamp: + secs: 1733882746 + nsecs: 15416622 +frame_id: "camera_color_optical_frame" +[INFO] [1733882746.454575]: received mask_obs with header seq: 236 +stamp: + secs: 1733882746 + nsecs: 455371618 +frame_id: '' and grasped_object_mask header seq: 32238 +stamp: + secs: 1733882746 + nsecs: 215255260 +frame_id: "camera_color_optical_frame" +[INFO] [1733882746.650796]: received mask_obs with header seq: 237 +stamp: + secs: 1733882746 + nsecs: 651590347 +frame_id: '' and grasped_object_mask header seq: 32244 +stamp: + secs: 1733882746 + nsecs: 415093899 +frame_id: "camera_color_optical_frame" +[INFO] [1733882746.850833]: received mask_obs with header seq: 238 +stamp: + secs: 1733882746 + nsecs: 851330041 +frame_id: '' and grasped_object_mask header seq: 32250 +stamp: + secs: 1733882746 + nsecs: 614925146 +frame_id: "camera_color_optical_frame" +[INFO] [1733882747.051500]: received mask_obs with header seq: 239 +stamp: + secs: 1733882747 + nsecs: 52041530 +frame_id: '' and grasped_object_mask header seq: 32256 +stamp: + secs: 1733882746 + nsecs: 814756870 +frame_id: "camera_color_optical_frame" +[INFO] [1733882747.253331]: received mask_obs with header seq: 240 +stamp: + secs: 1733882747 + nsecs: 254077434 +frame_id: '' and grasped_object_mask header seq: 32262 +stamp: + secs: 1733882747 + nsecs: 14587402 +frame_id: "camera_color_optical_frame" +[INFO] [1733882747.473652]: received mask_obs with header seq: 241 +stamp: + secs: 1733882747 + nsecs: 474446535 +frame_id: '' and grasped_object_mask header seq: 32268 +stamp: + secs: 1733882747 + nsecs: 214417458 +frame_id: "camera_color_optical_frame" +[INFO] [1733882747.668667]: received mask_obs with header seq: 242 +stamp: + secs: 1733882747 + nsecs: 669385433 +frame_id: '' and grasped_object_mask header seq: 32275 +stamp: + secs: 1733882747 + nsecs: 447554350 +frame_id: "camera_color_optical_frame" +[INFO] [1733882747.867553]: received mask_obs with header seq: 243 +stamp: + secs: 1733882747 + nsecs: 868296146 +frame_id: '' and grasped_object_mask header seq: 32281 +stamp: + secs: 1733882747 + nsecs: 647394657 +frame_id: "camera_color_optical_frame" +[INFO] [1733882748.089673]: received mask_obs with header seq: 244 +stamp: + secs: 1733882748 + nsecs: 90407848 +frame_id: '' and grasped_object_mask header seq: 32287 +stamp: + secs: 1733882747 + nsecs: 847235203 +frame_id: "camera_color_optical_frame" +[INFO] [1733882748.284217]: received mask_obs with header seq: 245 +stamp: + secs: 1733882748 + nsecs: 285010337 +frame_id: '' and grasped_object_mask header seq: 32293 +stamp: + secs: 1733882748 + nsecs: 47075748 +frame_id: "camera_color_optical_frame" +[INFO] [1733882748.478210]: received mask_obs with header seq: 246 +stamp: + secs: 1733882748 + nsecs: 478685617 +frame_id: '' and grasped_object_mask header seq: 32299 +stamp: + secs: 1733882748 + nsecs: 246916294 +frame_id: "camera_color_optical_frame" +[INFO] [1733882748.678099]: received mask_obs with header seq: 247 +stamp: + secs: 1733882748 + nsecs: 678304672 +frame_id: '' and grasped_object_mask header seq: 32305 +stamp: + secs: 1733882748 + nsecs: 446756601 +frame_id: "camera_color_optical_frame" +[INFO] [1733882748.873234]: received mask_obs with header seq: 248 +stamp: + secs: 1733882748 + nsecs: 873726606 +frame_id: '' and grasped_object_mask header seq: 32311 +stamp: + secs: 1733882748 + nsecs: 646587372 +frame_id: "camera_color_optical_frame" +[INFO] [1733882748.918962]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1733882749.070712]: received mask_obs with header seq: 249 +stamp: + secs: 1733882749 + nsecs: 70325136 +frame_id: '' and grasped_object_mask header seq: 32317 +stamp: + secs: 1733882748 + nsecs: 846416712 +frame_id: "camera_color_optical_frame" +[INFO] [1733882749.647767]: Stored episode 1. +[INFO] [1733882749.648048]: Saving video... +[INFO] [1733882749.862328]: Video saved! +Starting episode 1 +[INFO] [1733882749.880894]: resetting environment +[INFO] [1733882749.886885]: cleared current plan +[INFO] [1733882749.887112]: moving to home +[INFO] [1733882752.489864]: reached home +[INFO] [1733882752.491040]: reset action history +[INFO] [1733882755.198398]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +program exiting gracefully +proceeding to start episode! +[INFO] [1733882833.361588]: Saving video... +[INFO] [1733882833.362778]: Video saved! \ No newline at end of file diff --git a/210407/wandb/run-20241210_210414-ob7chcle/files/requirements.txt b/210407/wandb/run-20241210_210414-ob7chcle/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..192179a195956e252bda33f2342b4970a9de79d9 --- /dev/null +++ b/210407/wandb/run-20241210_210414-ob7chcle/files/requirements.txt @@ -0,0 +1,339 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.2 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/210407/wandb/run-20241210_210414-ob7chcle/files/wandb-metadata.json b/210407/wandb/run-20241210_210414-ob7chcle/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..1153151d1037201669e797d83fca4925fbb69bab --- /dev/null +++ b/210407/wandb/run-20241210_210414-ob7chcle/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2024-12-11T02:04:14.630135", + "startedAt": "2024-12-11T02:04:14.247012", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "4f7c0421eb84a7caf6bf5670f68777523362676f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 4124.996375000001, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.021, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.001, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.001, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.023, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 366.0494689941406 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71596908569336 + } +} diff --git a/210407/wandb/run-20241210_210414-ob7chcle/files/wandb-summary.json b/210407/wandb/run-20241210_210414-ob7chcle/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..fe8c538b1f9d80c8b5a567d24c03dddf7ec6ac56 --- /dev/null +++ b/210407/wandb/run-20241210_210414-ob7chcle/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "242331cb3a64f66021dfad95eeae9d0d8610c587d62363724138c291f7e496ce", "size": 1091529, "path": "media/videos/eval/0_eval_0_242331cb3a64f66021df.mp4"}, "global_step": 136, "_timestamp": 1733882833.365749, "_runtime": 179.1091718673706, "_step": 1, "eval/video": {"_type": "video-file", "sha256": "25445a41b5bbda0ed7e2e0845eaa5c96373705c5c738fe640c1ea70a07b4176c", "size": 262, "path": "media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4"}, "_wandb": {"runtime": 178}} \ No newline at end of file diff --git a/210407/wandb/run-20241210_210414-ob7chcle/logs/debug-internal.log b/210407/wandb/run-20241210_210414-ob7chcle/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..dd0fc984e9385549fd72db7ed4ef7b2dfdb78f56 --- /dev/null +++ b/210407/wandb/run-20241210_210414-ob7chcle/logs/debug-internal.log @@ -0,0 +1,510 @@ +2024-12-10 21:04:14,256 INFO StreamThr :600018 [internal.py:wandb_internal():85] W&B internal server running at pid: 600018, started at: 2024-12-10 21:04:14.256478 +2024-12-10 21:04:14,265 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status +2024-12-10 21:04:14,269 INFO WriterThread:600018 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/run-ob7chcle.wandb +2024-12-10 21:04:14,270 DEBUG SenderThread:600018 [sender.py:send():379] send: header +2024-12-10 21:04:14,286 DEBUG SenderThread:600018 [sender.py:send():379] send: run +2024-12-10 21:04:14,541 INFO SenderThread:600018 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files +2024-12-10 21:04:14,541 INFO SenderThread:600018 [sender.py:_start_run_threads():1188] run started: ob7chcle with start time 1733882654.256577 +2024-12-10 21:04:14,564 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: check_version +2024-12-10 21:04:14,564 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: check_version +2024-12-10 21:04:14,612 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: run_start +2024-12-10 21:04:14,617 DEBUG HandlerThread:600018 [system_info.py:__init__():26] System info init +2024-12-10 21:04:14,617 DEBUG HandlerThread:600018 [system_info.py:__init__():41] System info init done +2024-12-10 21:04:14,617 INFO HandlerThread:600018 [system_monitor.py:start():194] Starting system monitor +2024-12-10 21:04:14,617 INFO SystemMonitor:600018 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-10 21:04:14,617 INFO HandlerThread:600018 [system_monitor.py:probe():214] Collecting system info +2024-12-10 21:04:14,618 INFO SystemMonitor:600018 [interfaces.py:start():188] Started cpu monitoring +2024-12-10 21:04:14,618 INFO SystemMonitor:600018 [interfaces.py:start():188] Started disk monitoring +2024-12-10 21:04:14,619 INFO SystemMonitor:600018 [interfaces.py:start():188] Started gpu monitoring +2024-12-10 21:04:14,619 INFO SystemMonitor:600018 [interfaces.py:start():188] Started memory monitoring +2024-12-10 21:04:14,620 INFO SystemMonitor:600018 [interfaces.py:start():188] Started network monitoring +2024-12-10 21:04:14,630 DEBUG HandlerThread:600018 [system_info.py:probe():152] Probing system +2024-12-10 21:04:14,631 DEBUG HandlerThread:600018 [system_info.py:_probe_git():137] Probing git +2024-12-10 21:04:14,636 DEBUG HandlerThread:600018 [system_info.py:_probe_git():145] Probing git done +2024-12-10 21:04:14,636 DEBUG HandlerThread:600018 [system_info.py:probe():200] Probing system done +2024-12-10 21:04:14,636 DEBUG HandlerThread:600018 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-11T02:04:14.630135', 'startedAt': '2024-12-11T02:04:14.247012', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '4f7c0421eb84a7caf6bf5670f68777523362676f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4124.996375000001, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.021, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}, {'current': 4500.023, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 366.0494689941406}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71596908569336}} +2024-12-10 21:04:14,636 INFO HandlerThread:600018 [system_monitor.py:probe():224] Finished collecting system info +2024-12-10 21:04:14,636 INFO HandlerThread:600018 [system_monitor.py:probe():227] Publishing system info +2024-12-10 21:04:14,636 DEBUG HandlerThread:600018 [system_info.py:_save_code():44] Saving code +2024-12-10 21:04:14,638 DEBUG HandlerThread:600018 [system_info.py:_save_code():67] Saving code done +2024-12-10 21:04:14,638 DEBUG HandlerThread:600018 [system_info.py:_save_patches():84] Saving git patches +2024-12-10 21:04:14,703 DEBUG HandlerThread:600018 [system_info.py:_save_patches():126] Saving git patches done +2024-12-10 21:04:14,704 INFO HandlerThread:600018 [system_monitor.py:probe():229] Finished publishing system info +2024-12-10 21:04:14,707 DEBUG SenderThread:600018 [sender.py:send():379] send: files +2024-12-10 21:04:14,707 INFO SenderThread:600018 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-10 21:04:14,708 INFO SenderThread:600018 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-10 21:04:14,708 INFO SenderThread:600018 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-10 21:04:14,959 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: python_packages +2024-12-10 21:04:14,960 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: python_packages +2024-12-10 21:04:14,960 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:04:14,960 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:04:14,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:14,963 INFO HandlerThread:600018 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/tb" + save: true +} +control { + relay_id: "5cdf21b441a841eeb077b1d066a521dc" +} +_info { + stream_id: "ob7chcle" +} + +2024-12-10 21:04:14,963 DEBUG HandlerThread:600018 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-10 21:04:14,988 INFO wandb-upload_0:600018 [upload_job.py:push():130] Uploaded file /tmp/tmpr0qlu59lwandb/vuiipzpx-wandb-metadata.json +2024-12-10 21:04:15,027 DEBUG SenderThread:600018 [sender.py:send():379] send: telemetry +2024-12-10 21:04:15,027 DEBUG SenderThread:600018 [sender.py:send():379] send: tbrecord +2024-12-10 21:04:15,028 DEBUG SenderThread:600018 [sender.py:send():379] send: files +2024-12-10 21:04:15,028 INFO SenderThread:600018 [sender.py:_save_file():1454] saving file events.out.tfevents.1733882654.leonmkim-ROG-Strix-G15CS-G15CS.599894.0 with policy live +2024-12-10 21:04:15,030 INFO wandb-upload_1:600018 [upload_job.py:push():130] Uploaded file /tmp/tmpr0qlu59lwandb/4jll3ix3-code/FISH/eval_robot.py +2024-12-10 21:04:15,039 INFO wandb-upload_2:600018 [upload_job.py:push():130] Uploaded file /tmp/tmpr0qlu59lwandb/5pcafv76-diff.patch +2024-12-10 21:04:15,220 INFO wandb-upload_0:600018 [upload_job.py:push():130] Uploaded file /tmp/tmpr0qlu59lwandb/o3h98mw2-events.out.tfevents.1733882654.leonmkim-ROG-Strix-G15CS-G15CS.599894.0 +2024-12-10 21:04:15,543 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/requirements.txt +2024-12-10 21:04:15,543 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/code/FISH/eval_robot.py +2024-12-10 21:04:15,543 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/wandb-metadata.json +2024-12-10 21:04:15,543 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:04:15,543 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/events.out.tfevents.1733882654.leonmkim-ROG-Strix-G15CS-G15CS.599894.0 +2024-12-10 21:04:15,543 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/diff.patch +2024-12-10 21:04:15,543 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/code/FISH +2024-12-10 21:04:15,544 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/code +2024-12-10 21:04:15,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:16,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:17,542 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:04:17,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:18,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:19,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:04:19,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:20,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:21,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:22,962 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:23,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:24,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:04:24,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:25,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:26,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:27,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:28,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:29,960 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:04:29,960 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:04:30,002 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:30,045 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:04:30,965 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:31,965 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:32,965 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:33,965 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:34,965 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:35,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:04:35,965 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:36,965 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:37,966 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:38,966 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:39,966 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:40,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:04:40,966 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:41,966 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:42,966 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:43,966 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:44,960 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:04:44,960 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:04:45,001 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:45,967 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:46,018 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:04:46,550 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/config.yaml +2024-12-10 21:04:46,967 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:47,967 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:48,967 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:49,967 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:50,967 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:51,962 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:04:51,967 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:52,968 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:53,968 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:54,968 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:55,968 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:56,962 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:04:56,968 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:57,968 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:58,968 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:04:59,960 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:04:59,961 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:05:00,002 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:00,969 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:01,969 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:02,107 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:05:02,969 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:03,969 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:04,969 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:05,969 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:06,970 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:07,962 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:05:07,970 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:08,018 DEBUG SenderThread:600018 [sender.py:send():379] send: config +2024-12-10 21:05:08,970 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:09,556 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:09,970 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:10,970 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:11,556 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:11,971 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:12,970 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:13,449 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:05:13,557 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:13,971 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:14,620 DEBUG SystemMonitor:600018 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-10 21:05:14,621 DEBUG SenderThread:600018 [sender.py:send():379] send: stats +2024-12-10 21:05:14,960 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:05:14,961 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:05:15,002 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:15,558 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:15,971 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:16,971 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:17,558 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:17,971 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:18,600 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:05:18,971 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:19,559 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:19,559 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/config.yaml +2024-12-10 21:05:19,971 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:20,971 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:21,559 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:21,971 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:22,971 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:23,560 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:23,971 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:24,064 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:05:24,972 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:25,560 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:25,972 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:26,972 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:27,561 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:27,972 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:28,973 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:29,124 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:05:29,562 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:29,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:05:29,961 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:05:29,973 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:30,973 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:31,562 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:31,973 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:32,975 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:33,563 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:33,975 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:34,275 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:05:34,975 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:35,563 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:35,976 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:36,976 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:37,564 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:37,976 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:38,976 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:39,457 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:05:39,564 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:39,976 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:40,976 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:41,565 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:41,977 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:42,977 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:43,566 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:43,977 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:44,621 DEBUG SenderThread:600018 [sender.py:send():379] send: stats +2024-12-10 21:05:44,622 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:05:44,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:05:44,961 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:05:44,977 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:45,567 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:45,977 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:46,977 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:47,567 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:47,977 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:48,977 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:49,568 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:49,648 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:05:49,880 DEBUG SenderThread:600018 [sender.py:send():379] send: files +2024-12-10 21:05:49,880 INFO SenderThread:600018 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_242331cb3a64f66021df.mp4 with policy now +2024-12-10 21:05:49,880 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:05:49,881 DEBUG SenderThread:600018 [sender.py:send():379] send: history +2024-12-10 21:05:49,881 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:05:49,882 INFO SenderThread:600018 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:05:49,978 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:50,165 INFO wandb-upload_1:600018 [upload_job.py:push():130] Uploaded file /tmp/tmpr0qlu59lwandb/dx8d7p0q-media/videos/eval/0_eval_0_242331cb3a64f66021df.mp4 +2024-12-10 21:05:50,573 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/media/videos/eval/0_eval_0_242331cb3a64f66021df.mp4 +2024-12-10 21:05:50,573 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/wandb-summary.json +2024-12-10 21:05:50,573 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/media/videos/eval +2024-12-10 21:05:50,573 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/media/videos +2024-12-10 21:05:50,574 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/media +2024-12-10 21:05:50,912 INFO wandb-upload_0:600018 [upload_job.py:push():130] Uploaded file /tmp/tmpr0qlu59lwandb/64whhgw0-media/videos/eval/0_eval_0_242331cb3a64f66021df.mp4 +2024-12-10 21:05:50,978 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:51,573 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:51,978 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:52,978 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:53,574 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:53,979 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:54,963 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:05:54,978 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:55,978 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:56,978 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:57,575 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:05:57,978 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:58,979 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:05:59,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:05:59,961 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:06:00,001 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:00,045 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:06:00,979 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:01,979 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:02,979 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:03,979 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:04,979 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:05,963 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:06:05,979 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:06,979 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:07,979 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:08,980 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:09,980 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:10,980 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:11,963 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:06:11,980 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:12,980 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:13,980 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:14,622 DEBUG SenderThread:600018 [sender.py:send():379] send: stats +2024-12-10 21:06:14,961 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:06:14,961 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:06:15,001 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:15,980 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:16,980 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:17,057 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:06:17,980 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:18,981 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:19,981 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:20,981 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:21,981 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:22,963 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:06:22,981 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:23,981 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:24,981 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:25,982 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:26,982 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:27,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:06:27,982 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:28,982 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:29,962 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:06:29,962 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:06:30,005 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:30,982 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:31,982 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:32,982 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:33,045 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:06:33,983 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:34,983 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:35,983 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:36,983 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:37,983 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:38,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:06:38,983 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:39,983 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:40,984 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:41,984 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:42,984 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:43,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:06:43,985 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:44,622 DEBUG SenderThread:600018 [sender.py:send():379] send: stats +2024-12-10 21:06:44,962 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:06:44,962 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:06:45,006 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:45,986 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:46,986 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:47,987 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:48,986 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:49,027 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:06:49,987 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:50,986 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:51,987 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:52,987 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:53,987 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:54,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:06:54,987 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:55,987 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:56,987 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:57,987 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:58,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:06:59,962 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:06:59,962 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:07:00,005 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:00,048 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:07:00,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:01,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:02,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:03,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:04,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:05,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:07:05,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:06,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:07,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:08,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:09,989 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:10,964 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:07:10,989 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:11,989 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:12,999 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:13,402 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:07:13,402 DEBUG SenderThread:600018 [sender.py:send():379] send: files +2024-12-10 21:07:13,402 INFO SenderThread:600018 [sender.py:_save_file():1454] saving file media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 with policy now +2024-12-10 21:07:13,403 DEBUG SenderThread:600018 [sender.py:send():379] send: history +2024-12-10 21:07:13,403 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:07:13,403 INFO SenderThread:600018 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:07:13,404 DEBUG SenderThread:600018 [sender.py:send():379] send: exit +2024-12-10 21:07:13,404 INFO SenderThread:600018 [sender.py:send_exit():586] handling exit code: 0 +2024-12-10 21:07:13,404 INFO SenderThread:600018 [sender.py:send_exit():588] handling runtime: 178 +2024-12-10 21:07:13,404 INFO SenderThread:600018 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:07:13,404 INFO SenderThread:600018 [sender.py:send_exit():594] send defer +2024-12-10 21:07:13,404 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:13,404 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-10 21:07:13,405 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:13,405 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-10 21:07:13,405 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 1 +2024-12-10 21:07:13,405 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:13,405 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-10 21:07:13,406 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:13,406 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-10 21:07:13,406 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 2 +2024-12-10 21:07:13,407 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:13,407 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-10 21:07:13,407 INFO HandlerThread:600018 [system_monitor.py:finish():203] Stopping system monitor +2024-12-10 21:07:13,407 DEBUG SystemMonitor:600018 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-10 21:07:13,407 DEBUG SystemMonitor:600018 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-10 21:07:13,407 INFO HandlerThread:600018 [interfaces.py:finish():200] Joined cpu monitor +2024-12-10 21:07:13,408 INFO HandlerThread:600018 [interfaces.py:finish():200] Joined disk monitor +2024-12-10 21:07:13,456 INFO HandlerThread:600018 [interfaces.py:finish():200] Joined gpu monitor +2024-12-10 21:07:13,456 INFO HandlerThread:600018 [interfaces.py:finish():200] Joined memory monitor +2024-12-10 21:07:13,456 INFO HandlerThread:600018 [interfaces.py:finish():200] Joined network monitor +2024-12-10 21:07:13,456 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:13,456 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-10 21:07:13,456 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 3 +2024-12-10 21:07:13,456 DEBUG SenderThread:600018 [sender.py:send():379] send: stats +2024-12-10 21:07:13,456 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:13,457 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-10 21:07:13,457 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:13,457 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-10 21:07:13,457 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 4 +2024-12-10 21:07:13,457 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:13,457 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-10 21:07:13,599 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:07:13,600 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/wandb-summary.json +2024-12-10 21:07:13,600 INFO Thread-12 :600018 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 +2024-12-10 21:07:13,600 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/media/videos/eval +2024-12-10 21:07:13,607 INFO wandb-upload_1:600018 [upload_job.py:push():130] Uploaded file /tmp/tmpr0qlu59lwandb/ycl94f2h-media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 +2024-12-10 21:07:13,892 INFO wandb-upload_0:600018 [upload_job.py:push():130] Uploaded file /tmp/tmpr0qlu59lwandb/53aux0ik-media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 +2024-12-10 21:07:15,600 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:07:19,985 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:07:19,985 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:19,985 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:07:19,986 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-10 21:07:19,986 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: keepalive +2024-12-10 21:07:19,986 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 5 +2024-12-10 21:07:19,986 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:07:19,986 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:19,986 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-10 21:07:19,987 DEBUG SenderThread:600018 [sender.py:send():379] send: summary +2024-12-10 21:07:19,987 INFO SenderThread:600018 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:07:19,987 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:19,987 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-10 21:07:19,987 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 6 +2024-12-10 21:07:19,987 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:19,987 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-10 21:07:19,987 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:19,988 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-10 21:07:19,988 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 7 +2024-12-10 21:07:19,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:07:19,988 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:19,988 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-10 21:07:19,988 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:19,988 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-10 21:07:20,405 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:07:20,602 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/wandb-summary.json +2024-12-10 21:07:21,263 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 8 +2024-12-10 21:07:21,263 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:07:21,263 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:21,263 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-10 21:07:21,263 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:21,263 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-10 21:07:21,263 INFO SenderThread:600018 [job_builder.py:build():440] Attempting to build job artifact +2024-12-10 21:07:21,266 INFO SenderThread:600018 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-10 21:07:21,280 INFO SenderThread:600018 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-10 21:07:21,294 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 9 +2024-12-10 21:07:21,295 DEBUG SenderThread:600018 [sender.py:send():379] send: artifact +2024-12-10 21:07:21,295 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:21,296 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-10 21:07:21,405 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:07:21,602 INFO Thread-12 :600018 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:07:21,765 INFO wandb-upload_2:600018 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0up_8ql4 +2024-12-10 21:07:21,896 INFO wandb-upload_1:600018 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmphpmz9ew4 +2024-12-10 21:07:21,956 INFO wandb-upload_0:600018 [upload_job.py:push():88] Uploaded file /tmp/tmpbv2i4zjr/wandb-job.json +2024-12-10 21:07:22,564 INFO SenderThread:600018 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM2Njc3NTA2NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM2Njc2MTczOA==', 'versionIndex': 337}}} +2024-12-10 21:07:22,564 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:22,564 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-10 21:07:22,564 INFO SenderThread:600018 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-10 21:07:22,603 INFO SenderThread:600018 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files +2024-12-10 21:07:22,603 INFO SenderThread:600018 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/config.yaml config.yaml +2024-12-10 21:07:22,603 INFO SenderThread:600018 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/wandb-summary.json wandb-summary.json +2024-12-10 21:07:22,605 INFO SenderThread:600018 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/requirements.txt requirements.txt +2024-12-10 21:07:22,605 INFO SenderThread:600018 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/wandb-metadata.json wandb-metadata.json +2024-12-10 21:07:22,606 INFO SenderThread:600018 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/events.out.tfevents.1733882654.leonmkim-ROG-Strix-G15CS-G15CS.599894.0 events.out.tfevents.1733882654.leonmkim-ROG-Strix-G15CS-G15CS.599894.0 +2024-12-10 21:07:22,606 INFO SenderThread:600018 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log output.log +2024-12-10 21:07:22,607 INFO SenderThread:600018 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/diff.patch diff.patch +2024-12-10 21:07:22,608 INFO SenderThread:600018 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-10 21:07:22,610 INFO SenderThread:600018 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/media/videos/eval/0_eval_0_242331cb3a64f66021df.mp4 media/videos/eval/0_eval_0_242331cb3a64f66021df.mp4 +2024-12-10 21:07:22,610 INFO SenderThread:600018 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 media/videos/eval/video_1_25445a41b5bbda0ed7e2.mp4 +2024-12-10 21:07:22,610 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 10 +2024-12-10 21:07:22,610 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:07:22,610 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:22,610 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-10 21:07:22,610 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:22,610 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-10 21:07:22,610 INFO SenderThread:600018 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 21:07:22,797 INFO wandb-upload_0:600018 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/wandb-summary.json +2024-12-10 21:07:22,818 INFO wandb-upload_1:600018 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/requirements.txt +2024-12-10 21:07:22,829 INFO wandb-upload_2:600018 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/config.yaml +2024-12-10 21:07:22,893 INFO wandb-upload_3:600018 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/files/output.log +2024-12-10 21:07:23,093 INFO Thread-11 (_thread_body):600018 [sender.py:transition_state():614] send defer: 11 +2024-12-10 21:07:23,094 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:23,094 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-10 21:07:23,094 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:23,094 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-10 21:07:23,094 INFO SenderThread:600018 [file_pusher.py:join():175] waiting for file pusher +2024-12-10 21:07:23,094 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 12 +2024-12-10 21:07:23,095 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:23,095 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-10 21:07:23,095 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:23,095 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-10 21:07:23,095 INFO SenderThread:600018 [file_stream.py:finish():601] file stream finish called +2024-12-10 21:07:23,318 INFO SenderThread:600018 [file_stream.py:finish():605] file stream finish is done +2024-12-10 21:07:23,319 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 13 +2024-12-10 21:07:23,319 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:23,319 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-10 21:07:23,319 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:23,319 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-10 21:07:23,319 INFO SenderThread:600018 [sender.py:transition_state():614] send defer: 14 +2024-12-10 21:07:23,319 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:07:23,319 INFO HandlerThread:600018 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-10 21:07:23,319 DEBUG SenderThread:600018 [sender.py:send():379] send: final +2024-12-10 21:07:23,320 DEBUG SenderThread:600018 [sender.py:send():379] send: footer +2024-12-10 21:07:23,320 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: defer +2024-12-10 21:07:23,320 INFO SenderThread:600018 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-10 21:07:23,320 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:07:23,320 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: server_info +2024-12-10 21:07:23,320 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:07:23,321 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: server_info +2024-12-10 21:07:23,322 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:07:23,322 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: get_summary +2024-12-10 21:07:23,323 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-10 21:07:23,323 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:23,446 DEBUG SenderThread:600018 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:07:23,446 INFO MainThread:600018 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-10 21:07:23,446 INFO MainThread:600018 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-10 21:07:23,446 INFO MainThread:600018 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-10 21:07:23,446 DEBUG HandlerThread:600018 [handler.py:handle_request():158] handle_request: shutdown +2024-12-10 21:07:23,446 INFO HandlerThread:600018 [handler.py:finish():882] shutting down handler +2024-12-10 21:07:24,322 INFO WriterThread:600018 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/run-ob7chcle.wandb +2024-12-10 21:07:24,446 INFO SenderThread:600018 [sender.py:finish():1615] shutting down sender +2024-12-10 21:07:24,446 INFO SenderThread:600018 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 21:07:24,446 INFO SenderThread:600018 [file_pusher.py:join():175] waiting for file pusher diff --git a/210407/wandb/run-20241210_210414-ob7chcle/logs/debug.log b/210407/wandb/run-20241210_210414-ob7chcle/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4b3ee6e78ce69391bc1cdea3a993b48122fc9d53 --- /dev/null +++ b/210407/wandb/run-20241210_210414-ob7chcle/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-10 21:04:14,250 INFO MainThread:599894 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-10 21:04:14,250 INFO MainThread:599894 [wandb_setup.py:_flush():76] Configure stats pid to 599894 +2024-12-10 21:04:14,250 INFO MainThread:599894 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-10 21:04:14,250 INFO MainThread:599894 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/settings +2024-12-10 21:04:14,250 INFO MainThread:599894 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-10 21:04:14,250 INFO MainThread:599894 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-10 21:04:14,250 INFO MainThread:599894 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-10 21:04:14,250 INFO MainThread:599894 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-10 21:04:14,250 INFO MainThread:599894 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/logs/debug.log +2024-12-10 21:04:14,251 INFO MainThread:599894 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/wandb/run-20241210_210414-ob7chcle/logs/debug-internal.log +2024-12-10 21:04:14,251 INFO MainThread:599894 [wandb_init.py:init():569] calling init triggers +2024-12-10 21:04:14,251 INFO MainThread:599894 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '265626', 'wandb_notes': '265626_req_265827', 'eval': True, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim_left_to_right'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'obs_shape': [5, 240, 320], 'action_shape': [36, 7], 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'true_action_history': False, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt'} +2024-12-10 21:04:14,251 INFO MainThread:599894 [wandb_init.py:init():619] starting backend +2024-12-10 21:04:14,251 INFO MainThread:599894 [wandb_init.py:init():623] setting up manager +2024-12-10 21:04:14,255 INFO MainThread:599894 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-10 21:04:14,256 INFO MainThread:599894 [wandb_init.py:init():631] backend started and connected +2024-12-10 21:04:14,268 INFO MainThread:599894 [wandb_init.py:init():720] updated telemetry +2024-12-10 21:04:14,283 INFO MainThread:599894 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-10 21:04:14,563 INFO MainThread:599894 [wandb_run.py:_on_init():2435] communicating current version +2024-12-10 21:04:14,605 INFO MainThread:599894 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.0 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-10 21:04:14,606 INFO MainThread:599894 [wandb_init.py:init():804] starting run threads in backend +2024-12-10 21:04:14,960 INFO MainThread:599894 [wandb_run.py:_console_start():2413] atexit reg +2024-12-10 21:04:14,960 INFO MainThread:599894 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-10 21:04:14,960 INFO MainThread:599894 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-10 21:04:14,961 INFO MainThread:599894 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-10 21:04:14,962 INFO MainThread:599894 [wandb_init.py:init():847] run started, returning control to user process +2024-12-10 21:04:14,962 INFO MainThread:599894 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210407/tb, True +2024-12-10 21:05:08,018 INFO MainThread:599894 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-10 21:07:24,447 WARNING MsgRouterThr:599894 [router.py:message_loop():77] message_loop has been closed diff --git a/210407/wandb/run-20241210_210414-ob7chcle/run-ob7chcle.wandb b/210407/wandb/run-20241210_210414-ob7chcle/run-ob7chcle.wandb new file mode 100644 index 0000000000000000000000000000000000000000..26f94b157ea7b64987b2075d3dda3719fa7d22f0 Binary files /dev/null and b/210407/wandb/run-20241210_210414-ob7chcle/run-ob7chcle.wandb differ diff --git a/210732/.hydra/config.yaml b/210732/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d37c497bc2fe8883c44596565f059ac8e1269416 --- /dev/null +++ b/210732/.hydra/config.yaml @@ -0,0 +1,349 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '265626' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/210732/.hydra/hydra.yaml b/210732/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c224a99032d5141e463807f3ab03940ca9db27a2 --- /dev/null +++ b/210732/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${experiment_dir}/${now:%H%M%S} + sweep: + dir: ${experiment_dir}/${now:%H%M%S} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${experiment_dir}/${now:%H%M%S}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/210732/.hydra/overrides.yaml b/210732/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/210732/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/210732/episode_rosbags/aligned_depth_to_color_K.npy b/210732/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/210732/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/210732/episode_rosbags/cam_tf_world.npy b/210732/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/210732/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/210732/episode_rosbags/color_K.npy b/210732/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/210732/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/210732/episode_rosbags/depth_K.npy b/210732/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/210732/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/210732/episode_rosbags/episode_0_2024-12-10-21-08-09.rrd b/210732/episode_rosbags/episode_0_2024-12-10-21-08-09.rrd new file mode 100644 index 0000000000000000000000000000000000000000..bab5ae2b512825e9bb0f526ea3e8f1e2bd9edd3f --- /dev/null +++ b/210732/episode_rosbags/episode_0_2024-12-10-21-08-09.rrd @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f743d176718e27870c8f1cc08c5af9c9d912ab17ec9f7500e33cb7e2841c15cc +size 430113421 diff --git a/210732/episode_rosbags/episode_1_2024-12-10-21-09-04.rrd b/210732/episode_rosbags/episode_1_2024-12-10-21-09-04.rrd new file mode 100644 index 0000000000000000000000000000000000000000..cc94568782447fbfd73479788de8d214789b3bb9 --- /dev/null +++ b/210732/episode_rosbags/episode_1_2024-12-10-21-09-04.rrd @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92727b3a565edcb6701aea0bb47834e42788826854dabe18e1e1df75b61e1b4d +size 429295017 diff --git a/210732/episode_rosbags/episode_2_2024-12-10-21-09-45.rrd b/210732/episode_rosbags/episode_2_2024-12-10-21-09-45.rrd new file mode 100644 index 0000000000000000000000000000000000000000..b0c991689bd07e001b876a03c4cb2b368047e357 --- /dev/null +++ b/210732/episode_rosbags/episode_2_2024-12-10-21-09-45.rrd @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:657aeb4ef44c75ac3b6889a84ec80e32f8391eb1b9423ebd1fb4bcd9042acf3a +size 435761423 diff --git a/210732/episode_rosbags/episode_4_2024-12-10-21-11-07.rrd b/210732/episode_rosbags/episode_4_2024-12-10-21-11-07.rrd new file mode 100644 index 0000000000000000000000000000000000000000..854a1d609111b1ab291c0cd98937dcf49339e9d8 --- /dev/null +++ b/210732/episode_rosbags/episode_4_2024-12-10-21-11-07.rrd @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad603c969d81e0122e899bf67268e7106cbfd4983e086e5336bd80cea7ea36f3 +size 430463608 diff --git a/210732/eval_robot.log b/210732/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..0379d2f89d2013021a6f46ec4e8ae9e66cfccf50 --- /dev/null +++ b/210732/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-10 21:07:32,466][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-10 21:07:32,469][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/210732/eval_video/0_eval.mp4 b/210732/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0d3c10d9ddb7815435e0a47b00a0fe80e4aaed48 --- /dev/null +++ b/210732/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daf03ac4df433b6afa8025ee9f7d8ed0bed270796e3304bfc12b399ae0335495 +size 1152674 diff --git a/210732/eval_video/1_eval.mp4 b/210732/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bfad2054f1a0d394023b4e0092f4a36205e5d02b --- /dev/null +++ b/210732/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa72f11620e871983c64a5c2dc859cc7580356d19b495eb0134d04e0f3ef9263 +size 1175556 diff --git a/210732/eval_video/2_eval.mp4 b/210732/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..991e2dc0b0e1b0f12adde16e8c8795850574ccea --- /dev/null +++ b/210732/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b17305aedebc761a4898e8d4ff0fd8cd6ab34922d528b2e1833fe396b670cfbe +size 1170247 diff --git a/210732/eval_video/3_eval.mp4 b/210732/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2618a43d687b0201a7bb665c2aa48ddb8686fd2e --- /dev/null +++ b/210732/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49c63cb7694dee4c930a2f65bf4b4cf1d5025f26cca5462b29f9cd8b7b7aa953 +size 1171672 diff --git a/210732/eval_video/4_eval.mp4 b/210732/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..52111b7bdbb0273c254cea7dcc32571d222c2934 --- /dev/null +++ b/210732/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a244b0b09dc15d2eae1cd3aaa59edb276ea040384e1f9010b1b996ca7083d6e5 +size 1157328 diff --git a/210732/tb/events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 b/210732/tb/events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 new file mode 100644 index 0000000000000000000000000000000000000000..7a6057047c5589ab5d4895d6672360ae72f312ba --- /dev/null +++ b/210732/tb/events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e5ef6ff56f9b22c23ccfe96bb0744db4f224abb026891433b2baacf5e099cd6 +size 1103 diff --git a/210732/wandb/debug-internal.log b/210732/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ba4ae0369f190c7e02df864203e4dfd8110c033c --- /dev/null +++ b/210732/wandb/debug-internal.log @@ -0,0 +1,742 @@ +2024-12-10 21:07:38,622 INFO StreamThr :601021 [internal.py:wandb_internal():85] W&B internal server running at pid: 601021, started at: 2024-12-10 21:07:38.622053 +2024-12-10 21:07:38,623 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status +2024-12-10 21:07:38,634 INFO WriterThread:601021 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/run-r4m9fzdi.wandb +2024-12-10 21:07:38,634 DEBUG SenderThread:601021 [sender.py:send():379] send: header +2024-12-10 21:07:38,645 DEBUG SenderThread:601021 [sender.py:send():379] send: run +2024-12-10 21:07:38,918 INFO SenderThread:601021 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files +2024-12-10 21:07:38,918 INFO SenderThread:601021 [sender.py:_start_run_threads():1188] run started: r4m9fzdi with start time 1733882858.622519 +2024-12-10 21:07:38,936 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: check_version +2024-12-10 21:07:38,936 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: check_version +2024-12-10 21:07:38,984 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: run_start +2024-12-10 21:07:38,989 DEBUG HandlerThread:601021 [system_info.py:__init__():26] System info init +2024-12-10 21:07:38,989 DEBUG HandlerThread:601021 [system_info.py:__init__():41] System info init done +2024-12-10 21:07:38,989 INFO HandlerThread:601021 [system_monitor.py:start():194] Starting system monitor +2024-12-10 21:07:38,989 INFO SystemMonitor:601021 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-10 21:07:38,989 INFO HandlerThread:601021 [system_monitor.py:probe():214] Collecting system info +2024-12-10 21:07:38,989 INFO SystemMonitor:601021 [interfaces.py:start():188] Started cpu monitoring +2024-12-10 21:07:38,990 INFO SystemMonitor:601021 [interfaces.py:start():188] Started disk monitoring +2024-12-10 21:07:38,991 INFO SystemMonitor:601021 [interfaces.py:start():188] Started gpu monitoring +2024-12-10 21:07:38,991 INFO SystemMonitor:601021 [interfaces.py:start():188] Started memory monitoring +2024-12-10 21:07:38,992 INFO SystemMonitor:601021 [interfaces.py:start():188] Started network monitoring +2024-12-10 21:07:39,002 DEBUG HandlerThread:601021 [system_info.py:probe():152] Probing system +2024-12-10 21:07:39,003 DEBUG HandlerThread:601021 [system_info.py:_probe_git():137] Probing git +2024-12-10 21:07:39,008 DEBUG HandlerThread:601021 [system_info.py:_probe_git():145] Probing git done +2024-12-10 21:07:39,008 DEBUG HandlerThread:601021 [system_info.py:probe():200] Probing system done +2024-12-10 21:07:39,008 DEBUG HandlerThread:601021 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-11T02:07:39.002284', 'startedAt': '2024-12-11T02:07:38.614206', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '4f7c0421eb84a7caf6bf5670f68777523362676f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4312.5005, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.997, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.956, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 367.3990364074707}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71596908569336}} +2024-12-10 21:07:39,008 INFO HandlerThread:601021 [system_monitor.py:probe():224] Finished collecting system info +2024-12-10 21:07:39,008 INFO HandlerThread:601021 [system_monitor.py:probe():227] Publishing system info +2024-12-10 21:07:39,008 DEBUG HandlerThread:601021 [system_info.py:_save_code():44] Saving code +2024-12-10 21:07:39,010 DEBUG HandlerThread:601021 [system_info.py:_save_code():67] Saving code done +2024-12-10 21:07:39,010 DEBUG HandlerThread:601021 [system_info.py:_save_patches():84] Saving git patches +2024-12-10 21:07:39,074 DEBUG HandlerThread:601021 [system_info.py:_save_patches():126] Saving git patches done +2024-12-10 21:07:39,074 INFO HandlerThread:601021 [system_monitor.py:probe():229] Finished publishing system info +2024-12-10 21:07:39,077 DEBUG SenderThread:601021 [sender.py:send():379] send: files +2024-12-10 21:07:39,077 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-10 21:07:39,077 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-10 21:07:39,078 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-10 21:07:39,335 INFO wandb-upload_0:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/19y6xz09-wandb-metadata.json +2024-12-10 21:07:39,342 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: python_packages +2024-12-10 21:07:39,342 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: python_packages +2024-12-10 21:07:39,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:07:39,345 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:07:39,351 INFO wandb-upload_2:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/yyukg9ez-diff.patch +2024-12-10 21:07:39,386 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:39,386 INFO HandlerThread:601021 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/tb" + save: true +} +control { + relay_id: "0d9e67a43a6245129e461b3eb5978722" +} +_info { + stream_id: "r4m9fzdi" +} + +2024-12-10 21:07:39,386 DEBUG HandlerThread:601021 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-10 21:07:39,459 INFO wandb-upload_1:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/6dtumhkg-code/FISH/eval_robot.py +2024-12-10 21:07:39,463 DEBUG SenderThread:601021 [sender.py:send():379] send: telemetry +2024-12-10 21:07:39,463 DEBUG SenderThread:601021 [sender.py:send():379] send: tbrecord +2024-12-10 21:07:39,464 DEBUG SenderThread:601021 [sender.py:send():379] send: files +2024-12-10 21:07:39,464 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 with policy live +2024-12-10 21:07:39,666 INFO wandb-upload_0:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/0lfrfmoc-events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 +2024-12-10 21:07:39,920 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/code/FISH/eval_robot.py +2024-12-10 21:07:39,920 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/diff.patch +2024-12-10 21:07:39,920 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 +2024-12-10 21:07:39,920 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:07:39,920 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/requirements.txt +2024-12-10 21:07:39,920 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-metadata.json +2024-12-10 21:07:39,920 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/code +2024-12-10 21:07:39,920 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/code/FISH +2024-12-10 21:07:40,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:41,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:41,919 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:07:42,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:42,752 DEBUG SenderThread:601021 [sender.py:send():379] send: config +2024-12-10 21:07:43,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:43,760 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:07:43,920 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:07:44,345 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:45,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:45,920 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:07:46,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:47,345 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:47,921 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:07:48,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:49,343 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:07:49,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:49,921 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:07:50,345 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:51,345 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:51,922 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:07:52,345 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:53,346 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:54,342 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:07:54,342 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:07:54,385 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:54,435 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:07:55,346 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:56,346 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:57,346 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:58,346 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:59,346 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:00,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:00,346 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:01,346 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:02,346 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:03,347 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:04,347 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:05,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:05,347 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:06,347 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:07,347 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:07,926 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:08,347 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:09,342 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:08:09,343 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:08:09,385 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:10,348 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:10,376 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:10,927 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/config.yaml +2024-12-10 21:08:11,348 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:11,927 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:12,348 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:13,348 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:13,928 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:14,348 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:15,348 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:15,562 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:15,928 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:16,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:17,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:17,929 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:18,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:19,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:19,930 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:20,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:20,631 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:21,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:21,930 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:22,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:23,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:23,931 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:24,342 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:08:24,343 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:08:24,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:25,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:25,779 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:25,931 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:26,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:27,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:27,933 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:28,350 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:29,350 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:29,933 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:30,350 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:30,977 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:31,350 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:31,934 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:32,350 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:33,350 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:33,934 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:34,350 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:35,350 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:35,935 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:36,166 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:36,350 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:37,350 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:37,935 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:38,229 DEBUG SenderThread:601021 [sender.py:send():379] send: files +2024-12-10 21:08:38,229 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_daf03ac4df433b6afa80.mp4 with policy now +2024-12-10 21:08:38,229 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:08:38,230 DEBUG SenderThread:601021 [sender.py:send():379] send: history +2024-12-10 21:08:38,230 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:08:38,231 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:08:38,351 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:38,482 INFO wandb-upload_1:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/pmuvl3df-media/videos/eval/0_eval_0_daf03ac4df433b6afa80.mp4 +2024-12-10 21:08:38,936 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json +2024-12-10 21:08:38,936 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/0_eval_0_daf03ac4df433b6afa80.mp4 +2024-12-10 21:08:38,936 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval +2024-12-10 21:08:38,936 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos +2024-12-10 21:08:38,936 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media +2024-12-10 21:08:38,992 DEBUG SystemMonitor:601021 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-10 21:08:38,992 DEBUG SenderThread:601021 [sender.py:send():379] send: stats +2024-12-10 21:08:39,235 INFO wandb-upload_0:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/40lm6lse-media/videos/eval/0_eval_0_daf03ac4df433b6afa80.mp4 +2024-12-10 21:08:39,343 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:08:39,343 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:08:39,389 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:39,936 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:40,351 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:41,351 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:41,937 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:42,086 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:42,352 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:43,351 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:44,351 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:45,352 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:45,938 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:46,352 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:47,345 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:47,352 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:48,352 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:49,353 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:50,353 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:51,353 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:52,346 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:52,353 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:53,353 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:54,343 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:08:54,343 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:08:54,385 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:55,354 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:56,354 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:57,354 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:57,445 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:58,354 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:59,354 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:00,354 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:01,354 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:01,943 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 +2024-12-10 21:09:02,197 INFO wandb-upload_1:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/2q7yeh2p-events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 +2024-12-10 21:09:02,354 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:03,246 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:09:03,354 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:03,944 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:04,355 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:05,355 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:05,945 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:06,355 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:07,355 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:07,945 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:08,355 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:08,417 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:09:08,993 DEBUG SenderThread:601021 [sender.py:send():379] send: stats +2024-12-10 21:09:09,343 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:09:09,343 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:09:09,356 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:09,946 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:10,355 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:11,355 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:11,947 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:12,356 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:13,356 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:13,420 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:09:13,947 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:14,356 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:15,356 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:15,948 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:16,356 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:17,357 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:17,949 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:18,357 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:18,442 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:09:19,358 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:19,950 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:20,360 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:21,359 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:21,950 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:22,359 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:23,360 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:23,619 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:09:23,951 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:24,343 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:09:24,343 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:09:24,360 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:25,360 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:25,952 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:26,360 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:27,360 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:27,952 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:28,360 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:28,639 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:09:29,360 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:29,953 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:30,360 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:31,360 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:31,954 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:32,361 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:33,366 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:33,618 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:09:33,618 DEBUG SenderThread:601021 [sender.py:send():379] send: files +2024-12-10 21:09:33,619 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_fa72f11620e871983c64.mp4 with policy now +2024-12-10 21:09:33,619 DEBUG SenderThread:601021 [sender.py:send():379] send: history +2024-12-10 21:09:33,620 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:09:33,620 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:09:33,879 INFO wandb-upload_0:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/rlxqu1db-media/videos/eval/1_eval_1_fa72f11620e871983c64.mp4 +2024-12-10 21:09:33,955 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:33,955 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json +2024-12-10 21:09:33,955 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/1_eval_1_fa72f11620e871983c64.mp4 +2024-12-10 21:09:33,955 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval +2024-12-10 21:09:34,255 INFO wandb-upload_1:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/69opme6e-media/videos/eval/1_eval_1_fa72f11620e871983c64.mp4 +2024-12-10 21:09:34,351 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:09:34,363 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:35,371 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:35,955 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:36,364 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:37,367 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:37,956 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:38,369 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:38,993 DEBUG SenderThread:601021 [sender.py:send():379] send: stats +2024-12-10 21:09:39,343 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:09:39,343 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:09:39,364 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:39,480 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:09:39,957 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:40,364 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:41,365 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:41,909 DEBUG SenderThread:601021 [sender.py:send():379] send: history +2024-12-10 21:09:41,909 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:09:41,910 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:09:41,957 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json +2024-12-10 21:09:41,958 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 +2024-12-10 21:09:42,366 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:43,365 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:43,958 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:44,365 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:45,182 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:09:45,366 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:45,959 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:46,365 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:47,365 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:47,960 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:48,365 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:49,368 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:49,960 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:50,248 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:09:50,368 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:51,368 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:51,961 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:52,368 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:53,369 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:53,962 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:54,343 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:09:54,343 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:09:54,369 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:55,369 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:55,455 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:09:55,963 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:56,369 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:57,369 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:57,964 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:58,369 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:59,369 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:59,964 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:00,369 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:00,638 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:01,370 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:01,965 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:02,370 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:03,370 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:03,966 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:04,370 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:05,370 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:05,671 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:05,967 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:06,370 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:07,371 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:07,967 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:08,371 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:08,994 DEBUG SenderThread:601021 [sender.py:send():379] send: stats +2024-12-10 21:10:09,343 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:10:09,344 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:10:09,371 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:09,968 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:10,371 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:10,843 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:11,371 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:11,969 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:12,371 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:13,373 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:13,970 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:14,025 DEBUG SenderThread:601021 [sender.py:send():379] send: files +2024-12-10 21:10:14,025 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_b17305aedebc761a4898.mp4 with policy now +2024-12-10 21:10:14,026 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:10:14,027 DEBUG SenderThread:601021 [sender.py:send():379] send: history +2024-12-10 21:10:14,027 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:10:14,028 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:10:14,282 INFO wandb-upload_0:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/a215d27a-media/videos/eval/2_eval_2_b17305aedebc761a4898.mp4 +2024-12-10 21:10:14,374 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:14,970 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json +2024-12-10 21:10:14,970 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/2_eval_2_b17305aedebc761a4898.mp4 +2024-12-10 21:10:14,970 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval +2024-12-10 21:10:15,275 INFO wandb-upload_1:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/zmle0ig3-media/videos/eval/2_eval_2_b17305aedebc761a4898.mp4 +2024-12-10 21:10:15,388 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:15,970 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:16,353 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:16,376 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:17,378 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:17,971 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:18,389 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:19,379 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:20,380 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:21,387 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:21,690 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:21,972 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:22,387 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:22,672 DEBUG SenderThread:601021 [sender.py:send():379] send: history +2024-12-10 21:10:22,672 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:10:22,695 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:10:22,973 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json +2024-12-10 21:10:22,973 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 +2024-12-10 21:10:23,233 INFO wandb-upload_0:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/qp1en2ki-events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 +2024-12-10 21:10:23,384 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:23,973 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:24,347 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:10:24,347 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:10:24,389 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:25,393 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:25,974 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:26,386 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:26,987 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:27,387 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:27,975 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:28,386 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:29,387 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:29,975 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:30,387 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:31,387 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:31,976 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:32,011 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:32,387 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:33,387 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:33,977 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:34,387 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:35,387 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:35,978 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:36,387 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:37,171 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:37,388 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:37,978 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:38,388 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:38,994 DEBUG SenderThread:601021 [sender.py:send():379] send: stats +2024-12-10 21:10:39,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:10:39,344 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:10:39,388 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:39,979 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:40,388 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:41,388 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:41,980 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:42,176 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:42,388 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:43,388 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:43,981 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:44,388 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:45,388 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:45,982 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:46,389 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:47,263 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:47,389 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:47,982 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:48,389 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:49,389 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:49,983 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:50,389 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:51,390 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:51,984 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:52,390 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:52,404 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:53,390 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:53,985 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:54,352 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:10:54,352 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:10:54,394 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:55,036 DEBUG SenderThread:601021 [sender.py:send():379] send: files +2024-12-10 21:10:55,036 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_49c63cb7694dee4c930a.mp4 with policy now +2024-12-10 21:10:55,043 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:10:55,044 DEBUG SenderThread:601021 [sender.py:send():379] send: history +2024-12-10 21:10:55,044 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:10:55,044 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:10:55,288 INFO wandb-upload_1:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/d9c1a6dv-media/videos/eval/3_eval_3_49c63cb7694dee4c930a.mp4 +2024-12-10 21:10:55,393 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:55,985 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:55,986 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json +2024-12-10 21:10:55,986 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/3_eval_3_49c63cb7694dee4c930a.mp4 +2024-12-10 21:10:55,986 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval +2024-12-10 21:10:56,283 INFO wandb-upload_0:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/rmi7s53d-media/videos/eval/3_eval_3_49c63cb7694dee4c930a.mp4 +2024-12-10 21:10:56,399 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:57,401 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:57,872 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:58,405 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:59,406 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:59,987 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:00,410 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:01,409 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:01,987 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:02,413 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:03,405 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:11:03,449 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:03,718 DEBUG SenderThread:601021 [sender.py:send():379] send: history +2024-12-10 21:11:03,718 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:11:03,718 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:11:03,988 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:03,988 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json +2024-12-10 21:11:03,988 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 +2024-12-10 21:11:04,415 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:05,414 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:05,989 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:06,419 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:07,416 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:07,990 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:08,415 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:08,464 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:11:08,995 DEBUG SenderThread:601021 [sender.py:send():379] send: stats +2024-12-10 21:11:09,345 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:11:09,345 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:11:09,415 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:09,990 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:10,416 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:11,415 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:11,991 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:12,416 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:13,416 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:13,485 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:11:13,992 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:14,416 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:15,416 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:15,993 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:16,416 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:17,416 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:17,993 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:18,417 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:18,684 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:11:19,417 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:19,994 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:20,417 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:21,417 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:21,995 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:22,417 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:23,417 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:23,723 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:11:23,996 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:24,345 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:11:24,345 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:11:24,417 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:25,417 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:25,996 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:26,417 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:27,417 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:27,997 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:28,418 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:28,907 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:11:29,418 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:29,998 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:30,418 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:31,418 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:32,418 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:33,001 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:33,420 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:33,958 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:11:34,419 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:35,002 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:35,424 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:36,155 DEBUG SenderThread:601021 [sender.py:send():379] send: files +2024-12-10 21:11:36,155 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_a244b0b09dc15d2eae1c.mp4 with policy now +2024-12-10 21:11:36,158 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:11:36,158 DEBUG SenderThread:601021 [sender.py:send():379] send: history +2024-12-10 21:11:36,158 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:11:36,158 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:11:36,398 INFO wandb-upload_1:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/knvvnylb-media/videos/eval/4_eval_4_a244b0b09dc15d2eae1c.mp4 +2024-12-10 21:11:36,426 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:37,003 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:37,003 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json +2024-12-10 21:11:37,003 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/4_eval_4_a244b0b09dc15d2eae1c.mp4 +2024-12-10 21:11:37,003 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval +2024-12-10 21:11:37,309 INFO wandb-upload_0:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/c2ykdz2u-media/videos/eval/4_eval_4_a244b0b09dc15d2eae1c.mp4 +2024-12-10 21:11:37,424 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:38,424 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:38,995 DEBUG SenderThread:601021 [sender.py:send():379] send: stats +2024-12-10 21:11:38,996 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:11:39,003 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:39,357 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:11:39,357 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:11:39,427 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:40,449 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:41,004 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:41,445 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:42,447 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:43,446 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:44,205 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:11:44,250 DEBUG SenderThread:601021 [sender.py:send():379] send: exit +2024-12-10 21:11:44,250 INFO SenderThread:601021 [sender.py:send_exit():586] handling exit code: 0 +2024-12-10 21:11:44,250 INFO SenderThread:601021 [sender.py:send_exit():588] handling runtime: 245 +2024-12-10 21:11:44,250 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:11:44,251 INFO SenderThread:601021 [sender.py:send_exit():594] send defer +2024-12-10 21:11:44,251 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:44,251 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-10 21:11:44,251 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:44,251 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-10 21:11:44,251 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 1 +2024-12-10 21:11:44,251 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:44,251 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-10 21:11:44,251 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:44,251 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-10 21:11:44,251 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 2 +2024-12-10 21:11:44,251 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:44,251 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-10 21:11:44,251 INFO HandlerThread:601021 [system_monitor.py:finish():203] Stopping system monitor +2024-12-10 21:11:44,251 DEBUG SystemMonitor:601021 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-10 21:11:44,252 INFO HandlerThread:601021 [interfaces.py:finish():200] Joined cpu monitor +2024-12-10 21:11:44,252 DEBUG SystemMonitor:601021 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-10 21:11:44,252 INFO HandlerThread:601021 [interfaces.py:finish():200] Joined disk monitor +2024-12-10 21:11:44,304 INFO HandlerThread:601021 [interfaces.py:finish():200] Joined gpu monitor +2024-12-10 21:11:44,304 INFO HandlerThread:601021 [interfaces.py:finish():200] Joined memory monitor +2024-12-10 21:11:44,304 INFO HandlerThread:601021 [interfaces.py:finish():200] Joined network monitor +2024-12-10 21:11:44,304 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:44,305 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-10 21:11:44,305 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 3 +2024-12-10 21:11:44,305 DEBUG SenderThread:601021 [sender.py:send():379] send: stats +2024-12-10 21:11:44,305 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:44,305 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-10 21:11:44,305 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:44,305 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-10 21:11:44,305 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 4 +2024-12-10 21:11:44,305 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:44,305 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-10 21:11:45,006 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:45,006 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json +2024-12-10 21:11:45,006 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 +2024-12-10 21:11:45,246 INFO wandb-upload_1:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/r96vgupj-events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 +2024-12-10 21:11:47,006 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:50,307 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:11:50,308 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:11:50,308 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:50,308 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: keepalive +2024-12-10 21:11:50,308 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-10 21:11:50,308 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 5 +2024-12-10 21:11:50,308 DEBUG SenderThread:601021 [sender.py:send():379] send: history +2024-12-10 21:11:50,308 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:50,308 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:11:50,308 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-10 21:11:50,309 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:11:50,309 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:11:50,309 DEBUG SenderThread:601021 [sender.py:send():379] send: history +2024-12-10 21:11:50,309 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:11:50,313 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:11:50,313 DEBUG SenderThread:601021 [sender.py:send():379] send: summary +2024-12-10 21:11:50,313 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:11:50,313 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:50,314 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-10 21:11:50,314 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 6 +2024-12-10 21:11:50,314 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:50,314 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-10 21:11:50,314 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:50,314 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-10 21:11:50,314 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 7 +2024-12-10 21:11:50,314 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:11:50,314 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:50,314 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-10 21:11:50,314 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:50,314 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-10 21:11:51,008 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json +2024-12-10 21:11:51,251 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:11:52,132 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 8 +2024-12-10 21:11:52,132 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:11:52,132 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:52,133 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-10 21:11:52,133 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:52,133 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-10 21:11:52,133 INFO SenderThread:601021 [job_builder.py:build():440] Attempting to build job artifact +2024-12-10 21:11:52,136 INFO SenderThread:601021 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-10 21:11:52,151 INFO SenderThread:601021 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-10 21:11:52,160 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 9 +2024-12-10 21:11:52,160 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:52,160 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-10 21:11:52,160 DEBUG SenderThread:601021 [sender.py:send():379] send: artifact +2024-12-10 21:11:52,252 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:11:52,718 INFO wandb-upload_2:601021 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmphlyt7eoj +2024-12-10 21:11:52,983 INFO wandb-upload_1:601021 [upload_job.py:push():88] Uploaded file /tmp/tmpsi_q7yxg/wandb-job.json +2024-12-10 21:11:53,006 INFO wandb-upload_0:601021 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpk9mk35pr +2024-12-10 21:11:53,009 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:53,692 INFO SenderThread:601021 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM2Njc3OTY4NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM2Njc3NTA2NA==', 'versionIndex': 338}}} +2024-12-10 21:11:53,692 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:53,692 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-10 21:11:53,692 INFO SenderThread:601021 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-10 21:11:54,009 INFO SenderThread:601021 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files +2024-12-10 21:11:54,010 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/config.yaml config.yaml +2024-12-10 21:11:54,010 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json wandb-summary.json +2024-12-10 21:11:54,012 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/requirements.txt requirements.txt +2024-12-10 21:11:54,012 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-metadata.json wandb-metadata.json +2024-12-10 21:11:54,013 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 +2024-12-10 21:11:54,015 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log output.log +2024-12-10 21:11:54,015 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/diff.patch diff.patch +2024-12-10 21:11:54,016 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-10 21:11:54,017 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/0_eval_0_daf03ac4df433b6afa80.mp4 media/videos/eval/0_eval_0_daf03ac4df433b6afa80.mp4 +2024-12-10 21:11:54,017 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/4_eval_4_a244b0b09dc15d2eae1c.mp4 media/videos/eval/4_eval_4_a244b0b09dc15d2eae1c.mp4 +2024-12-10 21:11:54,017 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/3_eval_3_49c63cb7694dee4c930a.mp4 media/videos/eval/3_eval_3_49c63cb7694dee4c930a.mp4 +2024-12-10 21:11:54,017 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/2_eval_2_b17305aedebc761a4898.mp4 media/videos/eval/2_eval_2_b17305aedebc761a4898.mp4 +2024-12-10 21:11:54,017 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/1_eval_1_fa72f11620e871983c64.mp4 media/videos/eval/1_eval_1_fa72f11620e871983c64.mp4 +2024-12-10 21:11:54,017 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 10 +2024-12-10 21:11:54,017 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:11:54,017 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:54,017 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-10 21:11:54,018 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:54,018 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-10 21:11:54,018 INFO SenderThread:601021 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 21:11:54,206 INFO wandb-upload_0:601021 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json +2024-12-10 21:11:54,206 INFO wandb-upload_1:601021 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/requirements.txt +2024-12-10 21:11:54,248 INFO wandb-upload_2:601021 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/config.yaml +2024-12-10 21:11:54,252 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:11:54,253 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:11:54,282 INFO wandb-upload_3:601021 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:54,482 INFO Thread-11 (_thread_body):601021 [sender.py:transition_state():614] send defer: 11 +2024-12-10 21:11:54,482 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:54,482 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-10 21:11:54,483 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:54,483 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-10 21:11:54,483 INFO SenderThread:601021 [file_pusher.py:join():175] waiting for file pusher +2024-12-10 21:11:54,484 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 12 +2024-12-10 21:11:54,484 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:54,484 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-10 21:11:54,484 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:54,484 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-10 21:11:54,484 INFO SenderThread:601021 [file_stream.py:finish():601] file stream finish called +2024-12-10 21:11:54,670 INFO SenderThread:601021 [file_stream.py:finish():605] file stream finish is done +2024-12-10 21:11:54,670 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 13 +2024-12-10 21:11:54,670 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:54,671 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-10 21:11:54,671 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:54,671 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-10 21:11:54,671 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 14 +2024-12-10 21:11:54,671 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:54,671 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-10 21:11:54,671 DEBUG SenderThread:601021 [sender.py:send():379] send: final +2024-12-10 21:11:54,672 DEBUG SenderThread:601021 [sender.py:send():379] send: footer +2024-12-10 21:11:54,672 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:54,672 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-10 21:11:54,672 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:11:54,673 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:11:54,673 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: server_info +2024-12-10 21:11:54,673 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: server_info +2024-12-10 21:11:54,675 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:11:54,675 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: get_summary +2024-12-10 21:11:54,676 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-10 21:11:54,676 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:54,767 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:11:54,768 INFO MainThread:601021 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-10 21:11:54,768 INFO MainThread:601021 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-10 21:11:54,768 INFO MainThread:601021 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-10 21:11:54,768 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: shutdown +2024-12-10 21:11:54,768 INFO HandlerThread:601021 [handler.py:finish():882] shutting down handler +2024-12-10 21:11:55,675 INFO WriterThread:601021 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/run-r4m9fzdi.wandb +2024-12-10 21:11:55,768 INFO SenderThread:601021 [sender.py:finish():1615] shutting down sender +2024-12-10 21:11:55,768 INFO SenderThread:601021 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 21:11:55,768 INFO SenderThread:601021 [file_pusher.py:join():175] waiting for file pusher diff --git a/210732/wandb/debug.log b/210732/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9d046d8e75bfaab93f97ddbf392b64c702202e21 --- /dev/null +++ b/210732/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_setup.py:_flush():76] Configure stats pid to 600897 +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/settings +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/logs/debug.log +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/logs/debug-internal.log +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_init.py:init():569] calling init triggers +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '265626', 'wandb_notes': '265626_req_265827', 'eval': True, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim_left_to_right'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'obs_shape': [5, 240, 320], 'action_shape': [36, 7], 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'true_action_history': False, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt'} +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_init.py:init():619] starting backend +2024-12-10 21:07:38,618 INFO MainThread:600897 [wandb_init.py:init():623] setting up manager +2024-12-10 21:07:38,621 INFO MainThread:600897 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-10 21:07:38,622 INFO MainThread:600897 [wandb_init.py:init():631] backend started and connected +2024-12-10 21:07:38,633 INFO MainThread:600897 [wandb_init.py:init():720] updated telemetry +2024-12-10 21:07:38,642 INFO MainThread:600897 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-10 21:07:38,935 INFO MainThread:600897 [wandb_run.py:_on_init():2435] communicating current version +2024-12-10 21:07:38,977 INFO MainThread:600897 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.0 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-10 21:07:38,978 INFO MainThread:600897 [wandb_init.py:init():804] starting run threads in backend +2024-12-10 21:07:39,343 INFO MainThread:600897 [wandb_run.py:_console_start():2413] atexit reg +2024-12-10 21:07:39,343 INFO MainThread:600897 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-10 21:07:39,343 INFO MainThread:600897 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-10 21:07:39,344 INFO MainThread:600897 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-10 21:07:39,347 INFO MainThread:600897 [wandb_init.py:init():847] run started, returning control to user process +2024-12-10 21:07:39,347 INFO MainThread:600897 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/tb, True +2024-12-10 21:07:42,752 INFO MainThread:600897 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-10 21:11:55,771 WARNING MsgRouterThr:600897 [router.py:message_loop():77] message_loop has been closed diff --git a/210732/wandb/run-20241210_210738-r4m9fzdi/files/code/FISH/eval_robot.py b/210732/wandb/run-20241210_210738-r4m9fzdi/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..75a95b02db44bf1d6cdce4329109f52a0c4c1bc8 --- /dev/null +++ b/210732/wandb/run-20241210_210738-r4m9fzdi/files/code/FISH/eval_robot.py @@ -0,0 +1,512 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/210732/wandb/run-20241210_210738-r4m9fzdi/files/config.yaml b/210732/wandb/run-20241210_210738-r4m9fzdi/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..097fe1338b97666239fe1fabe3440b6f5e303c3c --- /dev/null +++ b/210732/wandb/run-20241210_210738-r4m9fzdi/files/config.yaml @@ -0,0 +1,966 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '265626' +wandb_notes: + desc: null + value: 265626_req_265827 +eval: + desc: null + value: true +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: + - hbm_twodim_left_to_right +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 0 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/265626 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + obs_shape: + - 5 + - 240 + - 320 + action_shape: + - 36 + - 7 + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + policy_frequency: 5 + interpolation_frequency: 25 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + policy_frequency: 5 + interpolation_frequency: 25 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +true_action_history: + desc: null + value: false +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1733882858 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 16 + - 23 + - 35 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/210732/wandb/run-20241210_210738-r4m9fzdi/files/diff.patch b/210732/wandb/run-20241210_210738-r4m9fzdi/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..e955a119f01c29164eba90675e14f23cac4e9500 --- /dev/null +++ b/210732/wandb/run-20241210_210738-r4m9fzdi/files/diff.patch @@ -0,0 +1,56 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a9568e3..aa52de5 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -123,6 +123,11 @@ bc_weight_type: 'qfilter' # linear, qfilter + + # Load weights + load_checkpoint: ${agent.load_checkpoint} ++# 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act ++# full dataset ++wandb_run_id: "265626" # RGBD+Mask+act history ++# wandb_run_id: "265628" # RGBD+Mask+contact+act history ++ + # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # full dataset + # 6 step action history +@@ -135,7 +140,7 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '264772_0' # RGBD+act history + # wandb_run_id: '264775_0' # RGBD+mask+act history + # wandb_run_id: '264776_0' # RGBD+contact+act history +-wandb_run_id: '264777_0' # RGBD+mask+contact+act history ++# wandb_run_id: '264777_0' # RGBD+mask+contact+act history + + # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 16 held out demos +@@ -151,7 +156,7 @@ wandb_run_id: '264777_0' # RGBD+mask+contact+act history + # wandb_run_id: '250380_0' # RGBD+mask+contact+act history + # wandb_run_id: '250370_0' # RGBD+contact+act history + +-true_action_history: true ++true_action_history: false + wandb_notes: null + + checkpoint_epoch: 12000 # for depth + all contact +diff --git a/gym-envs/gym_envs/envs/hardware/franka.py b/gym-envs/gym_envs/envs/hardware/franka.py +index 26f8ae3..e3031af 100644 +--- a/gym-envs/gym_envs/envs/hardware/franka.py ++++ b/gym-envs/gym_envs/envs/hardware/franka.py +@@ -176,7 +176,7 @@ class Plan: + return interpolated_action_history + + def get_action_plan_timestamps(self): +- start_timestamp = self.ros_msg_plan.obs_header.stamp.to_sec() ++ start_timestamp = self.ros_msg_plan.obs_anchor_header.stamp.to_sec() + action_plan_timestamps = np.arange(self.trajectory_length)*self.ros_msg_plan.action_period + start_timestamp + return action_plan_timestamps + +@@ -627,7 +627,7 @@ class Franka: + # # append the gripper actions to the end of the 6d actions + # plan_dict['action_trajectory'] = np.concatenate((plan_dict['action_trajectory'], np.array(plan.gripper_action_trajectory).astype(np.float32)), axis=1) + # plan_dict['action_period'] = plan.action_period +- self.plan_queue.add(plan.header, plan.obs_header.stamp, plan_dataclass) ++ self.plan_queue.add(plan.header, plan.obs_anchor_header.stamp, plan_dataclass) + else: + rospy.logwarn('In process of resetting, ignoring plan') + pass diff --git a/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/0_eval_0_daf03ac4df433b6afa80.mp4 b/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/0_eval_0_daf03ac4df433b6afa80.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0d3c10d9ddb7815435e0a47b00a0fe80e4aaed48 --- /dev/null +++ b/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/0_eval_0_daf03ac4df433b6afa80.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daf03ac4df433b6afa8025ee9f7d8ed0bed270796e3304bfc12b399ae0335495 +size 1152674 diff --git a/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/1_eval_1_fa72f11620e871983c64.mp4 b/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/1_eval_1_fa72f11620e871983c64.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bfad2054f1a0d394023b4e0092f4a36205e5d02b --- /dev/null +++ b/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/1_eval_1_fa72f11620e871983c64.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa72f11620e871983c64a5c2dc859cc7580356d19b495eb0134d04e0f3ef9263 +size 1175556 diff --git a/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/2_eval_2_b17305aedebc761a4898.mp4 b/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/2_eval_2_b17305aedebc761a4898.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..991e2dc0b0e1b0f12adde16e8c8795850574ccea --- /dev/null +++ b/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/2_eval_2_b17305aedebc761a4898.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b17305aedebc761a4898e8d4ff0fd8cd6ab34922d528b2e1833fe396b670cfbe +size 1170247 diff --git a/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/3_eval_3_49c63cb7694dee4c930a.mp4 b/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/3_eval_3_49c63cb7694dee4c930a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2618a43d687b0201a7bb665c2aa48ddb8686fd2e --- /dev/null +++ b/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/3_eval_3_49c63cb7694dee4c930a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49c63cb7694dee4c930a2f65bf4b4cf1d5025f26cca5462b29f9cd8b7b7aa953 +size 1171672 diff --git a/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/4_eval_4_a244b0b09dc15d2eae1c.mp4 b/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/4_eval_4_a244b0b09dc15d2eae1c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..52111b7bdbb0273c254cea7dcc32571d222c2934 --- /dev/null +++ b/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/4_eval_4_a244b0b09dc15d2eae1c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a244b0b09dc15d2eae1cd3aaa59edb276ea040384e1f9010b1b996ca7083d6e5 +size 1157328 diff --git a/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log b/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..783bdc9c06f37f52b97fe69dc4cdce02c0724280 --- /dev/null +++ b/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log @@ -0,0 +1,6321 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1733882862.752859]: resetting environment +[INFO] [1733882862.756469]: cleared current plan +[INFO] [1733882862.757632]: moving to home +[INFO] [1733882863.760039]: reached home +[INFO] [1733882863.765802]: reset action history +[INFO] [1733882866.475482]: environment reset +Starting episode 0 +[INFO] [1733882866.477988]: resetting environment +[INFO] [1733882866.481390]: cleared current plan +[INFO] [1733882866.482381]: moving to home +[INFO] [1733882867.484633]: reached home +[INFO] [1733882867.486382]: reset action history +[INFO] [1733882870.194085]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1733882885.888030]: resetting environment +[INFO] [1733882885.897994]: cleared current plan +[INFO] [1733882885.899077]: moving to home +[INFO] [1733882886.901461]: reached home +[INFO] [1733882886.903465]: reset action history +[INFO] [1733882889.610740]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/episode_rosbags/episode_0_2024-12-10-21-08-09.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-600897.log +started roslaunch server http://158.130.50.37:46671/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [601204] +process[print_text-2]: started with pid [601228] +process[rosbag_record-3]: started with pid [601229] +started bagging! +[INFO] [1733882890.372104]: received mask_obs with header seq: 250 +stamp: + secs: 1733882890 + nsecs: 372682809 +frame_id: '' and grasped_object_mask header seq: 36559 +stamp: + secs: 1733882890 + nsecs: 126775503 +frame_id: "camera_color_optical_frame" +[INFO] [1733882890.564631]: received mask_obs with header seq: 251 +stamp: + secs: 1733882890 + nsecs: 564475059 +frame_id: '' and grasped_object_mask header seq: 36565 +stamp: + secs: 1733882890 + nsecs: 326361179 +frame_id: "camera_color_optical_frame" +[INFO] [1733882890.740571]: received mask_obs with header seq: 252 +stamp: + secs: 1733882890 + nsecs: 740827798 +frame_id: '' and grasped_object_mask header seq: 36571 +stamp: + secs: 1733882890 + nsecs: 525936127 +frame_id: "camera_color_optical_frame" +[INFO] [1733882890.954771]: received mask_obs with header seq: 253 +stamp: + secs: 1733882890 + nsecs: 955078840 +frame_id: '' and grasped_object_mask header seq: 36577 +stamp: + secs: 1733882890 + nsecs: 726065159 +frame_id: "camera_color_optical_frame" +[INFO] [1733882891.152497]: received mask_obs with header seq: 254 +stamp: + secs: 1733882891 + nsecs: 153238773 +frame_id: '' and grasped_object_mask header seq: 36583 +stamp: + secs: 1733882890 + nsecs: 927003384 +frame_id: "camera_color_optical_frame" +[INFO] [1733882891.360220]: received mask_obs with header seq: 255 +stamp: + secs: 1733882891 + nsecs: 360868930 +frame_id: '' and grasped_object_mask header seq: 36589 +stamp: + secs: 1733882891 + nsecs: 128029585 +frame_id: "camera_color_optical_frame" +[INFO] [1733882891.559123]: received mask_obs with header seq: 256 +stamp: + secs: 1733882891 + nsecs: 559930562 +frame_id: '' and grasped_object_mask header seq: 36595 +stamp: + secs: 1733882891 + nsecs: 329143763 +frame_id: "camera_color_optical_frame" +[INFO] [1733882891.763784]: received mask_obs with header seq: 257 +stamp: + secs: 1733882891 + nsecs: 764599084 +frame_id: '' and grasped_object_mask header seq: 36601 +stamp: + secs: 1733882891 + nsecs: 530346155 +frame_id: "camera_color_optical_frame" +[INFO] [1733882891.959907]: received mask_obs with header seq: 258 +stamp: + secs: 1733882891 + nsecs: 960643291 +frame_id: '' and grasped_object_mask header seq: 36607 +stamp: + secs: 1733882891 + nsecs: 731007576 +frame_id: "camera_color_optical_frame" +[INFO] [1733882892.162333]: received mask_obs with header seq: 259 +stamp: + secs: 1733882892 + nsecs: 163057327 +frame_id: '' and grasped_object_mask header seq: 36613 +stamp: + secs: 1733882891 + nsecs: 930850983 +frame_id: "camera_color_optical_frame" +[INFO] [1733882892.365324]: received mask_obs with header seq: 260 +stamp: + secs: 1733882892 + nsecs: 365970611 +frame_id: '' and grasped_object_mask header seq: 36619 +stamp: + secs: 1733882892 + nsecs: 130695105 +frame_id: "camera_color_optical_frame" +[INFO] [1733882892.576344]: received mask_obs with header seq: 261 +stamp: + secs: 1733882892 + nsecs: 577203750 +frame_id: '' and grasped_object_mask header seq: 36625 +stamp: + secs: 1733882892 + nsecs: 330539227 +frame_id: "camera_color_optical_frame" +[INFO] [1733882892.762006]: received mask_obs with header seq: 262 +stamp: + secs: 1733882892 + nsecs: 762750148 +frame_id: '' and grasped_object_mask header seq: 36631 +stamp: + secs: 1733882892 + nsecs: 530383825 +frame_id: "camera_color_optical_frame" +[INFO] [1733882892.966766]: received mask_obs with header seq: 263 +stamp: + secs: 1733882892 + nsecs: 967580080 +frame_id: '' and grasped_object_mask header seq: 36637 +stamp: + secs: 1733882892 + nsecs: 730226517 +frame_id: "camera_color_optical_frame" +[INFO] [1733882893.165637]: received mask_obs with header seq: 264 +stamp: + secs: 1733882893 + nsecs: 166277647 +frame_id: '' and grasped_object_mask header seq: 36643 +stamp: + secs: 1733882892 + nsecs: 930067539 +frame_id: "camera_color_optical_frame" +[INFO] [1733882893.364310]: received mask_obs with header seq: 265 +stamp: + secs: 1733882893 + nsecs: 364749670 +frame_id: '' and grasped_object_mask header seq: 36649 +stamp: + secs: 1733882893 + nsecs: 129907846 +frame_id: "camera_color_optical_frame" +[INFO] [1733882893.561682]: received mask_obs with header seq: 266 +stamp: + secs: 1733882893 + nsecs: 562445640 +frame_id: '' and grasped_object_mask header seq: 36655 +stamp: + secs: 1733882893 + nsecs: 329748392 +frame_id: "camera_color_optical_frame" +[INFO] [1733882893.763680]: received mask_obs with header seq: 267 +stamp: + secs: 1733882893 + nsecs: 764261007 +frame_id: '' and grasped_object_mask header seq: 36661 +stamp: + secs: 1733882893 + nsecs: 529588699 +frame_id: "camera_color_optical_frame" +[INFO] [1733882893.969227]: received mask_obs with header seq: 268 +stamp: + secs: 1733882893 + nsecs: 969892263 +frame_id: '' and grasped_object_mask header seq: 36667 +stamp: + secs: 1733882893 + nsecs: 729427814 +frame_id: "camera_color_optical_frame" +[INFO] [1733882894.176356]: received mask_obs with header seq: 269 +stamp: + secs: 1733882894 + nsecs: 177217721 +frame_id: '' and grasped_object_mask header seq: 36673 +stamp: + secs: 1733882893 + nsecs: 929265022 +frame_id: "camera_color_optical_frame" +[INFO] [1733882894.374159]: received mask_obs with header seq: 270 +stamp: + secs: 1733882894 + nsecs: 374959945 +frame_id: '' and grasped_object_mask header seq: 36679 +stamp: + secs: 1733882894 + nsecs: 129103422 +frame_id: "camera_color_optical_frame" +[INFO] [1733882894.563635]: received mask_obs with header seq: 271 +stamp: + secs: 1733882894 + nsecs: 564286231 +frame_id: '' and grasped_object_mask header seq: 36685 +stamp: + secs: 1733882894 + nsecs: 328940630 +frame_id: "camera_color_optical_frame" +[INFO] [1733882894.761653]: received mask_obs with header seq: 272 +stamp: + secs: 1733882894 + nsecs: 762474536 +frame_id: '' and grasped_object_mask header seq: 36691 +stamp: + secs: 1733882894 + nsecs: 528777838 +frame_id: "camera_color_optical_frame" +[INFO] [1733882894.972122]: received mask_obs with header seq: 273 +stamp: + secs: 1733882894 + nsecs: 972851991 +frame_id: '' and grasped_object_mask header seq: 36697 +stamp: + secs: 1733882894 + nsecs: 728616238 +frame_id: "camera_color_optical_frame" +[INFO] [1733882895.169398]: received mask_obs with header seq: 274 +stamp: + secs: 1733882895 + nsecs: 170227766 +frame_id: '' and grasped_object_mask header seq: 36703 +stamp: + secs: 1733882894 + nsecs: 928456545 +frame_id: "camera_color_optical_frame" +[INFO] [1733882895.362255]: received mask_obs with header seq: 275 +stamp: + secs: 1733882895 + nsecs: 362183570 +frame_id: '' and grasped_object_mask header seq: 36709 +stamp: + secs: 1733882895 + nsecs: 128296852 +frame_id: "camera_color_optical_frame" +[INFO] [1733882895.561536]: received mask_obs with header seq: 276 +stamp: + secs: 1733882895 + nsecs: 561524629 +frame_id: '' and grasped_object_mask header seq: 36715 +stamp: + secs: 1733882895 + nsecs: 328138113 +frame_id: "camera_color_optical_frame" +[INFO] [1733882895.774717]: received mask_obs with header seq: 277 +stamp: + secs: 1733882895 + nsecs: 775389671 +frame_id: '' and grasped_object_mask header seq: 36721 +stamp: + secs: 1733882895 + nsecs: 527978659 +frame_id: "camera_color_optical_frame" +[INFO] [1733882895.971663]: received mask_obs with header seq: 278 +stamp: + secs: 1733882895 + nsecs: 972259521 +frame_id: '' and grasped_object_mask header seq: 36727 +stamp: + secs: 1733882895 + nsecs: 727818251 +frame_id: "camera_color_optical_frame" +[INFO] [1733882896.162612]: received mask_obs with header seq: 279 +stamp: + secs: 1733882896 + nsecs: 163231849 +frame_id: '' and grasped_object_mask header seq: 36730 +stamp: + secs: 1733882895 + nsecs: 827736855 +frame_id: "camera_color_optical_frame" +[INFO] [1733882896.363376]: received mask_obs with header seq: 280 +stamp: + secs: 1733882896 + nsecs: 364046335 +frame_id: '' and grasped_object_mask header seq: 36739 +stamp: + secs: 1733882896 + nsecs: 127494097 +frame_id: "camera_color_optical_frame" +[INFO] [1733882896.566545]: received mask_obs with header seq: 281 +stamp: + secs: 1733882896 + nsecs: 567151784 +frame_id: '' and grasped_object_mask header seq: 36745 +stamp: + secs: 1733882896 + nsecs: 327332258 +frame_id: "camera_color_optical_frame" +[INFO] [1733882896.763021]: received mask_obs with header seq: 282 +stamp: + secs: 1733882896 + nsecs: 763863325 +frame_id: '' and grasped_object_mask header seq: 36751 +stamp: + secs: 1733882896 + nsecs: 527170181 +frame_id: "camera_color_optical_frame" +[INFO] [1733882896.966588]: received mask_obs with header seq: 283 +stamp: + secs: 1733882896 + nsecs: 967272758 +frame_id: '' and grasped_object_mask header seq: 36757 +stamp: + secs: 1733882896 + nsecs: 727011681 +frame_id: "camera_color_optical_frame" +[INFO] [1733882897.161948]: received mask_obs with header seq: 284 +stamp: + secs: 1733882897 + nsecs: 162785530 +frame_id: '' and grasped_object_mask header seq: 36763 +stamp: + secs: 1733882896 + nsecs: 926857471 +frame_id: "camera_color_optical_frame" +[INFO] [1733882897.370499]: received mask_obs with header seq: 285 +stamp: + secs: 1733882897 + nsecs: 371047496 +frame_id: '' and grasped_object_mask header seq: 36769 +stamp: + secs: 1733882897 + nsecs: 126703024 +frame_id: "camera_color_optical_frame" +[INFO] [1733882897.562387]: received mask_obs with header seq: 286 +stamp: + secs: 1733882897 + nsecs: 563098430 +frame_id: '' and grasped_object_mask header seq: 36775 +stamp: + secs: 1733882897 + nsecs: 326549292 +frame_id: "camera_color_optical_frame" +[INFO] [1733882897.765867]: received mask_obs with header seq: 287 +stamp: + secs: 1733882897 + nsecs: 766703367 +frame_id: '' and grasped_object_mask header seq: 36781 +stamp: + secs: 1733882897 + nsecs: 526395559 +frame_id: "camera_color_optical_frame" +[INFO] [1733882897.957731]: received mask_obs with header seq: 288 +stamp: + secs: 1733882897 + nsecs: 958576202 +frame_id: '' and grasped_object_mask header seq: 36787 +stamp: + secs: 1733882897 + nsecs: 726239920 +frame_id: "camera_color_optical_frame" +[INFO] [1733882898.172695]: received mask_obs with header seq: 289 +stamp: + secs: 1733882898 + nsecs: 173218965 +frame_id: '' and grasped_object_mask header seq: 36793 +stamp: + secs: 1733882897 + nsecs: 926080942 +frame_id: "camera_color_optical_frame" +[INFO] [1733882898.367862]: received mask_obs with header seq: 290 +stamp: + secs: 1733882898 + nsecs: 368688583 +frame_id: '' and grasped_object_mask header seq: 36799 +stamp: + secs: 1733882898 + nsecs: 125921011 +frame_id: "camera_color_optical_frame" +[INFO] [1733882898.566643]: received mask_obs with header seq: 291 +stamp: + secs: 1733882898 + nsecs: 567239284 +frame_id: '' and grasped_object_mask header seq: 36805 +stamp: + secs: 1733882898 + nsecs: 325761318 +frame_id: "camera_color_optical_frame" +[INFO] [1733882898.781791]: received mask_obs with header seq: 292 +stamp: + secs: 1733882898 + nsecs: 782111883 +frame_id: '' and grasped_object_mask header seq: 36811 +stamp: + secs: 1733882898 + nsecs: 525601387 +frame_id: "camera_color_optical_frame" +[INFO] [1733882899.000230]: received mask_obs with header seq: 293 +stamp: + secs: 1733882899 + nsecs: 1026153 +frame_id: '' and grasped_object_mask header seq: 36818 +stamp: + secs: 1733882898 + nsecs: 758746386 +frame_id: "camera_color_optical_frame" +[INFO] [1733882899.215929]: received mask_obs with header seq: 294 +stamp: + secs: 1733882899 + nsecs: 216701269 +frame_id: '' and grasped_object_mask header seq: 36824 +stamp: + secs: 1733882898 + nsecs: 958581448 +frame_id: "camera_color_optical_frame" +[INFO] [1733882899.417971]: received mask_obs with header seq: 295 +stamp: + secs: 1733882899 + nsecs: 418407678 +frame_id: '' and grasped_object_mask header seq: 36831 +stamp: + secs: 1733882899 + nsecs: 191722870 +frame_id: "camera_color_optical_frame" +[INFO] [1733882899.632377]: received mask_obs with header seq: 296 +stamp: + secs: 1733882899 + nsecs: 633173704 +frame_id: '' and grasped_object_mask header seq: 36837 +stamp: + secs: 1733882899 + nsecs: 391557932 +frame_id: "camera_color_optical_frame" +[INFO] [1733882899.829815]: received mask_obs with header seq: 297 +stamp: + secs: 1733882899 + nsecs: 830617189 +frame_id: '' and grasped_object_mask header seq: 36843 +stamp: + secs: 1733882899 + nsecs: 591392517 +frame_id: "camera_color_optical_frame" +[INFO] [1733882900.017088]: received mask_obs with header seq: 298 +stamp: + secs: 1733882900 + nsecs: 17582416 +frame_id: '' and grasped_object_mask header seq: 36849 +stamp: + secs: 1733882899 + nsecs: 791229248 +frame_id: "camera_color_optical_frame" +[INFO] [1733882900.218982]: received mask_obs with header seq: 299 +stamp: + secs: 1733882900 + nsecs: 219765901 +frame_id: '' and grasped_object_mask header seq: 36855 +stamp: + secs: 1733882899 + nsecs: 991066933 +frame_id: "camera_color_optical_frame" +[INFO] [1733882900.420369]: received mask_obs with header seq: 300 +stamp: + secs: 1733882900 + nsecs: 421041250 +frame_id: '' and grasped_object_mask header seq: 36861 +stamp: + secs: 1733882900 + nsecs: 190904379 +frame_id: "camera_color_optical_frame" +[INFO] [1733882900.631269]: received mask_obs with header seq: 301 +stamp: + secs: 1733882900 + nsecs: 632033586 +frame_id: '' and grasped_object_mask header seq: 36867 +stamp: + secs: 1733882900 + nsecs: 390742779 +frame_id: "camera_color_optical_frame" +[INFO] [1733882900.835787]: received mask_obs with header seq: 302 +stamp: + secs: 1733882900 + nsecs: 830025672 +frame_id: '' and grasped_object_mask header seq: 36873 +stamp: + secs: 1733882900 + nsecs: 590579987 +frame_id: "camera_color_optical_frame" +[INFO] [1733882901.053130]: received mask_obs with header seq: 303 +stamp: + secs: 1733882901 + nsecs: 53790807 +frame_id: '' and grasped_object_mask header seq: 36880 +stamp: + secs: 1733882900 + nsecs: 823732376 +frame_id: "camera_color_optical_frame" +[INFO] [1733882901.250477]: received mask_obs with header seq: 304 +stamp: + secs: 1733882901 + nsecs: 250642538 +frame_id: '' and grasped_object_mask header seq: 36886 +stamp: + secs: 1733882901 + nsecs: 23582458 +frame_id: "camera_color_optical_frame" +[INFO] [1733882901.445876]: received mask_obs with header seq: 305 +stamp: + secs: 1733882901 + nsecs: 446416139 +frame_id: '' and grasped_object_mask header seq: 36892 +stamp: + secs: 1733882901 + nsecs: 223431826 +frame_id: "camera_color_optical_frame" +[INFO] [1733882901.641335]: received mask_obs with header seq: 306 +stamp: + secs: 1733882901 + nsecs: 642256975 +frame_id: '' and grasped_object_mask header seq: 36898 +stamp: + secs: 1733882901 + nsecs: 423281908 +frame_id: "camera_color_optical_frame" +[INFO] [1733882901.844096]: received mask_obs with header seq: 307 +stamp: + secs: 1733882901 + nsecs: 844434738 +frame_id: '' and grasped_object_mask header seq: 36904 +stamp: + secs: 1733882901 + nsecs: 623132229 +frame_id: "camera_color_optical_frame" +[INFO] [1733882902.031596]: received mask_obs with header seq: 308 +stamp: + secs: 1733882902 + nsecs: 32460689 +frame_id: '' and grasped_object_mask header seq: 36909 +stamp: + secs: 1733882901 + nsecs: 789665937 +frame_id: "camera_color_optical_frame" +[INFO] [1733882902.223421]: received mask_obs with header seq: 309 +stamp: + secs: 1733882902 + nsecs: 224047899 +frame_id: '' and grasped_object_mask header seq: 36915 +stamp: + secs: 1733882901 + nsecs: 989502907 +frame_id: "camera_color_optical_frame" +[INFO] [1733882902.419737]: received mask_obs with header seq: 310 +stamp: + secs: 1733882902 + nsecs: 420635461 +frame_id: '' and grasped_object_mask header seq: 36921 +stamp: + secs: 1733882902 + nsecs: 189339876 +frame_id: "camera_color_optical_frame" +[INFO] [1733882902.615265]: received mask_obs with header seq: 311 +stamp: + secs: 1733882902 + nsecs: 615178585 +frame_id: '' and grasped_object_mask header seq: 36927 +stamp: + secs: 1733882902 + nsecs: 389176607 +frame_id: "camera_color_optical_frame" +[INFO] [1733882902.812707]: received mask_obs with header seq: 312 +stamp: + secs: 1733882902 + nsecs: 812711477 +frame_id: '' and grasped_object_mask header seq: 36933 +stamp: + secs: 1733882902 + nsecs: 589013100 +frame_id: "camera_color_optical_frame" +[INFO] [1733882903.004848]: received mask_obs with header seq: 313 +stamp: + secs: 1733882903 + nsecs: 5152702 +frame_id: '' and grasped_object_mask header seq: 36939 +stamp: + secs: 1733882902 + nsecs: 788851500 +frame_id: "camera_color_optical_frame" +[INFO] [1733882903.213670]: received mask_obs with header seq: 314 +stamp: + secs: 1733882903 + nsecs: 213760137 +frame_id: '' and grasped_object_mask header seq: 36945 +stamp: + secs: 1733882902 + nsecs: 988689661 +frame_id: "camera_color_optical_frame" +[INFO] [1733882903.405131]: received mask_obs with header seq: 315 +stamp: + secs: 1733882903 + nsecs: 405372381 +frame_id: '' and grasped_object_mask header seq: 36951 +stamp: + secs: 1733882903 + nsecs: 188527584 +frame_id: "camera_color_optical_frame" +[INFO] [1733882903.603997]: received mask_obs with header seq: 316 +stamp: + secs: 1733882903 + nsecs: 604352235 +frame_id: '' and grasped_object_mask header seq: 36956 +stamp: + secs: 1733882903 + nsecs: 355059624 +frame_id: "camera_color_optical_frame" +[INFO] [1733882903.798170]: received mask_obs with header seq: 317 +stamp: + secs: 1733882903 + nsecs: 799007177 +frame_id: '' and grasped_object_mask header seq: 36962 +stamp: + secs: 1733882903 + nsecs: 554897785 +frame_id: "camera_color_optical_frame" +[INFO] [1733882904.000701]: received mask_obs with header seq: 318 +stamp: + secs: 1733882904 + nsecs: 1349687 +frame_id: '' and grasped_object_mask header seq: 36968 +stamp: + secs: 1733882903 + nsecs: 755251646 +frame_id: "camera_color_optical_frame" +[INFO] [1733882904.191468]: received mask_obs with header seq: 319 +stamp: + secs: 1733882904 + nsecs: 192139863 +frame_id: '' and grasped_object_mask header seq: 36974 +stamp: + secs: 1733882903 + nsecs: 957393408 +frame_id: "camera_color_optical_frame" +[INFO] [1733882904.382650]: received mask_obs with header seq: 320 +stamp: + secs: 1733882904 + nsecs: 383405923 +frame_id: '' and grasped_object_mask header seq: 36980 +stamp: + secs: 1733882904 + nsecs: 159621000 +frame_id: "camera_color_optical_frame" +[INFO] [1733882904.591271]: received mask_obs with header seq: 321 +stamp: + secs: 1733882904 + nsecs: 592051267 +frame_id: '' and grasped_object_mask header seq: 36986 +stamp: + secs: 1733882904 + nsecs: 361935616 +frame_id: "camera_color_optical_frame" +[INFO] [1733882904.790820]: received mask_obs with header seq: 322 +stamp: + secs: 1733882904 + nsecs: 791425704 +frame_id: '' and grasped_object_mask header seq: 36992 +stamp: + secs: 1733882904 + nsecs: 564337730 +frame_id: "camera_color_optical_frame" +[INFO] [1733882904.977273]: received mask_obs with header seq: 323 +stamp: + secs: 1733882904 + nsecs: 978125810 +frame_id: '' and grasped_object_mask header seq: 36998 +stamp: + secs: 1733882904 + nsecs: 766168833 +frame_id: "camera_color_optical_frame" +[INFO] [1733882905.176549]: received mask_obs with header seq: 324 +stamp: + secs: 1733882905 + nsecs: 177268981 +frame_id: '' and grasped_object_mask header seq: 37004 +stamp: + secs: 1733882904 + nsecs: 965786219 +frame_id: "camera_color_optical_frame" +[INFO] [1733882905.380181]: received mask_obs with header seq: 325 +stamp: + secs: 1733882905 + nsecs: 381057024 +frame_id: '' and grasped_object_mask header seq: 37010 +stamp: + secs: 1733882905 + nsecs: 165391922 +frame_id: "camera_color_optical_frame" +[INFO] [1733882905.578445]: received mask_obs with header seq: 326 +stamp: + secs: 1733882905 + nsecs: 579317331 +frame_id: '' and grasped_object_mask header seq: 37016 +stamp: + secs: 1733882905 + nsecs: 364986420 +frame_id: "camera_color_optical_frame" +[INFO] [1733882905.778808]: received mask_obs with header seq: 327 +stamp: + secs: 1733882905 + nsecs: 779598951 +frame_id: '' and grasped_object_mask header seq: 37022 +stamp: + secs: 1733882905 + nsecs: 564568996 +frame_id: "camera_color_optical_frame" +[INFO] [1733882905.987371]: received mask_obs with header seq: 328 +stamp: + secs: 1733882905 + nsecs: 988243341 +frame_id: '' and grasped_object_mask header seq: 37028 +stamp: + secs: 1733882905 + nsecs: 764151335 +frame_id: "camera_color_optical_frame" +[INFO] [1733882906.181314]: received mask_obs with header seq: 329 +stamp: + secs: 1733882906 + nsecs: 182195901 +frame_id: '' and grasped_object_mask header seq: 37034 +stamp: + secs: 1733882905 + nsecs: 963761330 +frame_id: "camera_color_optical_frame" +[INFO] [1733882906.387722]: received mask_obs with header seq: 330 +stamp: + secs: 1733882906 + nsecs: 388431549 +frame_id: '' and grasped_object_mask header seq: 37040 +stamp: + secs: 1733882906 + nsecs: 163358450 +frame_id: "camera_color_optical_frame" +[INFO] [1733882906.595434]: received mask_obs with header seq: 331 +stamp: + secs: 1733882906 + nsecs: 596266746 +frame_id: '' and grasped_object_mask header seq: 37046 +stamp: + secs: 1733882906 + nsecs: 362943888 +frame_id: "camera_color_optical_frame" +[INFO] [1733882906.785367]: received mask_obs with header seq: 332 +stamp: + secs: 1733882906 + nsecs: 786175012 +frame_id: '' and grasped_object_mask header seq: 37052 +stamp: + secs: 1733882906 + nsecs: 562517643 +frame_id: "camera_color_optical_frame" +[INFO] [1733882906.982637]: received mask_obs with header seq: 333 +stamp: + secs: 1733882906 + nsecs: 983497381 +frame_id: '' and grasped_object_mask header seq: 37058 +stamp: + secs: 1733882906 + nsecs: 762091875 +frame_id: "camera_color_optical_frame" +[INFO] [1733882907.174228]: received mask_obs with header seq: 334 +stamp: + secs: 1733882907 + nsecs: 175121545 +frame_id: '' and grasped_object_mask header seq: 37064 +stamp: + secs: 1733882906 + nsecs: 961695910 +frame_id: "camera_color_optical_frame" +[INFO] [1733882907.384769]: received mask_obs with header seq: 335 +stamp: + secs: 1733882907 + nsecs: 385527610 +frame_id: '' and grasped_object_mask header seq: 37070 +stamp: + secs: 1733882907 + nsecs: 161288738 +frame_id: "camera_color_optical_frame" +[INFO] [1733882907.580993]: received mask_obs with header seq: 336 +stamp: + secs: 1733882907 + nsecs: 581912517 +frame_id: '' and grasped_object_mask header seq: 37076 +stamp: + secs: 1733882907 + nsecs: 360868931 +frame_id: "camera_color_optical_frame" +[INFO] [1733882907.774900]: received mask_obs with header seq: 337 +stamp: + secs: 1733882907 + nsecs: 775722980 +frame_id: '' and grasped_object_mask header seq: 37082 +stamp: + secs: 1733882907 + nsecs: 560436726 +frame_id: "camera_color_optical_frame" +[INFO] [1733882907.972482]: received mask_obs with header seq: 338 +stamp: + secs: 1733882907 + nsecs: 973308563 +frame_id: '' and grasped_object_mask header seq: 37088 +stamp: + secs: 1733882907 + nsecs: 760006905 +frame_id: "camera_color_optical_frame" +[INFO] [1733882908.167233]: received mask_obs with header seq: 339 +stamp: + secs: 1733882908 + nsecs: 167834997 +frame_id: '' and grasped_object_mask header seq: 37094 +stamp: + secs: 1733882907 + nsecs: 959615469 +frame_id: "camera_color_optical_frame" +[INFO] [1733882908.375277]: received mask_obs with header seq: 340 +stamp: + secs: 1733882908 + nsecs: 375484466 +frame_id: '' and grasped_object_mask header seq: 37100 +stamp: + secs: 1733882908 + nsecs: 159212589 +frame_id: "camera_color_optical_frame" +[INFO] [1733882908.563977]: received mask_obs with header seq: 341 +stamp: + secs: 1733882908 + nsecs: 564808607 +frame_id: '' and grasped_object_mask header seq: 37106 +stamp: + secs: 1733882908 + nsecs: 358798742 +frame_id: "camera_color_optical_frame" +[INFO] [1733882908.767584]: received mask_obs with header seq: 342 +stamp: + secs: 1733882908 + nsecs: 767478942 +frame_id: '' and grasped_object_mask header seq: 37112 +stamp: + secs: 1733882908 + nsecs: 558372498 +frame_id: "camera_color_optical_frame" +[INFO] [1733882908.979250]: received mask_obs with header seq: 343 +stamp: + secs: 1733882908 + nsecs: 979114055 +frame_id: '' and grasped_object_mask header seq: 37118 +stamp: + secs: 1733882908 + nsecs: 757955551 +frame_id: "camera_color_optical_frame" +[INFO] [1733882909.172397]: received mask_obs with header seq: 344 +stamp: + secs: 1733882909 + nsecs: 172824144 +frame_id: '' and grasped_object_mask header seq: 37124 +stamp: + secs: 1733882908 + nsecs: 957569599 +frame_id: "camera_color_optical_frame" +[INFO] [1733882909.374471]: received mask_obs with header seq: 345 +stamp: + secs: 1733882909 + nsecs: 375308275 +frame_id: '' and grasped_object_mask header seq: 37130 +stamp: + secs: 1733882909 + nsecs: 157172203 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -121100857 for nearest: 1733882909235662267 - target: 1733882909356763124 at idx 249 +[INFO] [1733882909.576413]: received mask_obs with header seq: 346 +stamp: + secs: 1733882909 + nsecs: 576492071 +frame_id: '' and grasped_object_mask header seq: 37136 +stamp: + secs: 1733882909 + nsecs: 356763124 +frame_id: "camera_color_optical_frame" +[INFO] [1733882909.772291]: received mask_obs with header seq: 347 +stamp: + secs: 1733882909 + nsecs: 773130178 +frame_id: '' and grasped_object_mask header seq: 37142 +stamp: + secs: 1733882909 + nsecs: 556343794 +frame_id: "camera_color_optical_frame" +[INFO] [1733882909.971176]: received mask_obs with header seq: 348 +stamp: + secs: 1733882909 + nsecs: 971240043 +frame_id: '' and grasped_object_mask header seq: 37148 +stamp: + secs: 1733882909 + nsecs: 755921125 +frame_id: "camera_color_optical_frame" +[INFO] [1733882910.165251]: received mask_obs with header seq: 349 +stamp: + secs: 1733882910 + nsecs: 165909290 +frame_id: '' and grasped_object_mask header seq: 37154 +stamp: + secs: 1733882909 + nsecs: 955533028 +frame_id: "camera_color_optical_frame" +[INFO] [1733882910.376805]: received mask_obs with header seq: 350 +stamp: + secs: 1733882910 + nsecs: 377678394 +frame_id: '' and grasped_object_mask header seq: 37160 +stamp: + secs: 1733882910 + nsecs: 155133009 +frame_id: "camera_color_optical_frame" +[INFO] [1733882910.579294]: received mask_obs with header seq: 351 +stamp: + secs: 1733882910 + nsecs: 580051183 +frame_id: '' and grasped_object_mask header seq: 37166 +stamp: + secs: 1733882910 + nsecs: 354721785 +frame_id: "camera_color_optical_frame" +[INFO] [1733882910.765726]: received mask_obs with header seq: 352 +stamp: + secs: 1733882910 + nsecs: 765935182 +frame_id: '' and grasped_object_mask header seq: 37172 +stamp: + secs: 1733882910 + nsecs: 554298639 +frame_id: "camera_color_optical_frame" +[INFO] [1733882910.977386]: received mask_obs with header seq: 353 +stamp: + secs: 1733882910 + nsecs: 977952003 +frame_id: '' and grasped_object_mask header seq: 37178 +stamp: + secs: 1733882910 + nsecs: 753875017 +frame_id: "camera_color_optical_frame" +[INFO] [1733882911.184651]: received mask_obs with header seq: 354 +stamp: + secs: 1733882911 + nsecs: 185465812 +frame_id: '' and grasped_object_mask header seq: 37184 +stamp: + secs: 1733882910 + nsecs: 953488350 +frame_id: "camera_color_optical_frame" +[INFO] [1733882911.406232]: received mask_obs with header seq: 355 +stamp: + secs: 1733882911 + nsecs: 406520843 +frame_id: '' and grasped_object_mask header seq: 37190 +stamp: + secs: 1733882911 + nsecs: 153090239 +frame_id: "camera_color_optical_frame" +[INFO] [1733882911.603804]: received mask_obs with header seq: 356 +stamp: + secs: 1733882911 + nsecs: 604671955 +frame_id: '' and grasped_object_mask header seq: 37197 +stamp: + secs: 1733882911 + nsecs: 385945082 +frame_id: "camera_color_optical_frame" +[INFO] [1733882911.817853]: received mask_obs with header seq: 357 +stamp: + secs: 1733882911 + nsecs: 818582773 +frame_id: '' and grasped_object_mask header seq: 37203 +stamp: + secs: 1733882911 + nsecs: 585521936 +frame_id: "camera_color_optical_frame" +[INFO] [1733882912.015494]: received mask_obs with header seq: 358 +stamp: + secs: 1733882912 + nsecs: 16354084 +frame_id: '' and grasped_object_mask header seq: 37209 +stamp: + secs: 1733882911 + nsecs: 785104752 +frame_id: "camera_color_optical_frame" +[INFO] [1733882912.218407]: received mask_obs with header seq: 359 +stamp: + secs: 1733882912 + nsecs: 219321966 +frame_id: '' and grasped_object_mask header seq: 37215 +stamp: + secs: 1733882911 + nsecs: 984710693 +frame_id: "camera_color_optical_frame" +[INFO] [1733882912.402435]: received mask_obs with header seq: 360 +stamp: + secs: 1733882912 + nsecs: 403218746 +frame_id: '' and grasped_object_mask header seq: 37221 +stamp: + secs: 1733882912 + nsecs: 184304953 +frame_id: "camera_color_optical_frame" +[INFO] [1733882912.611099]: received mask_obs with header seq: 361 +stamp: + secs: 1733882912 + nsecs: 611695528 +frame_id: '' and grasped_object_mask header seq: 37227 +stamp: + secs: 1733882912 + nsecs: 383887529 +frame_id: "camera_color_optical_frame" +[INFO] [1733882912.801178]: received mask_obs with header seq: 362 +stamp: + secs: 1733882912 + nsecs: 802033185 +frame_id: '' and grasped_object_mask header seq: 37233 +stamp: + secs: 1733882912 + nsecs: 583458900 +frame_id: "camera_color_optical_frame" +[INFO] [1733882912.998722]: received mask_obs with header seq: 363 +stamp: + secs: 1733882912 + nsecs: 999580860 +frame_id: '' and grasped_object_mask header seq: 37239 +stamp: + secs: 1733882912 + nsecs: 783040285 +frame_id: "camera_color_optical_frame" +[INFO] [1733882913.196401]: received mask_obs with header seq: 364 +stamp: + secs: 1733882913 + nsecs: 197151660 +frame_id: '' and grasped_object_mask header seq: 37245 +stamp: + secs: 1733882912 + nsecs: 982655525 +frame_id: "camera_color_optical_frame" +[INFO] [1733882913.387284]: received mask_obs with header seq: 365 +stamp: + secs: 1733882913 + nsecs: 387284994 +frame_id: '' and grasped_object_mask header seq: 37250 +stamp: + secs: 1733882913 + nsecs: 148992538 +frame_id: "camera_color_optical_frame" +[INFO] [1733882913.583481]: received mask_obs with header seq: 366 +stamp: + secs: 1733882913 + nsecs: 584244728 +frame_id: '' and grasped_object_mask header seq: 37256 +stamp: + secs: 1733882913 + nsecs: 348587751 +frame_id: "camera_color_optical_frame" +[INFO] [1733882913.777028]: received mask_obs with header seq: 367 +stamp: + secs: 1733882913 + nsecs: 777947902 +frame_id: '' and grasped_object_mask header seq: 37262 +stamp: + secs: 1733882913 + nsecs: 548170805 +frame_id: "camera_color_optical_frame" +[INFO] [1733882913.982288]: received mask_obs with header seq: 368 +stamp: + secs: 1733882913 + nsecs: 983128309 +frame_id: '' and grasped_object_mask header seq: 37268 +stamp: + secs: 1733882913 + nsecs: 747752190 +frame_id: "camera_color_optical_frame" +[INFO] [1733882914.166204]: received mask_obs with header seq: 369 +stamp: + secs: 1733882914 + nsecs: 167113780 +frame_id: '' and grasped_object_mask header seq: 37274 +stamp: + secs: 1733882913 + nsecs: 947369814 +frame_id: "camera_color_optical_frame" +[INFO] [1733882914.365263]: received mask_obs with header seq: 370 +stamp: + secs: 1733882914 + nsecs: 366147756 +frame_id: '' and grasped_object_mask header seq: 37280 +stamp: + secs: 1733882914 + nsecs: 146975994 +frame_id: "camera_color_optical_frame" +[INFO] [1733882914.570348]: received mask_obs with header seq: 371 +stamp: + secs: 1733882914 + nsecs: 571158885 +frame_id: '' and grasped_object_mask header seq: 37286 +stamp: + secs: 1733882914 + nsecs: 346570730 +frame_id: "camera_color_optical_frame" +[INFO] [1733882914.770191]: received mask_obs with header seq: 372 +stamp: + secs: 1733882914 + nsecs: 771034955 +frame_id: '' and grasped_object_mask header seq: 37292 +stamp: + secs: 1733882914 + nsecs: 546155214 +frame_id: "camera_color_optical_frame" +[INFO] [1733882914.969356]: received mask_obs with header seq: 373 +stamp: + secs: 1733882914 + nsecs: 970164060 +frame_id: '' and grasped_object_mask header seq: 37298 +stamp: + secs: 1733882914 + nsecs: 745735884 +frame_id: "camera_color_optical_frame" +[INFO] [1733882915.165805]: received mask_obs with header seq: 374 +stamp: + secs: 1733882915 + nsecs: 165838956 +frame_id: '' and grasped_object_mask header seq: 37304 +stamp: + secs: 1733882914 + nsecs: 945346832 +frame_id: "camera_color_optical_frame" +[INFO] [1733882915.370215]: received mask_obs with header seq: 375 +stamp: + secs: 1733882915 + nsecs: 371028900 +frame_id: '' and grasped_object_mask header seq: 37310 +stamp: + secs: 1733882915 + nsecs: 144946337 +frame_id: "camera_color_optical_frame" +[INFO] [1733882915.571049]: received mask_obs with header seq: 376 +stamp: + secs: 1733882915 + nsecs: 571883201 +frame_id: '' and grasped_object_mask header seq: 37316 +stamp: + secs: 1733882915 + nsecs: 344533920 +frame_id: "camera_color_optical_frame" +[INFO] [1733882915.766290]: received mask_obs with header seq: 377 +stamp: + secs: 1733882915 + nsecs: 766150712 +frame_id: '' and grasped_object_mask header seq: 37322 +stamp: + secs: 1733882915 + nsecs: 544109583 +frame_id: "camera_color_optical_frame" +[INFO] [1733882915.966348]: received mask_obs with header seq: 378 +stamp: + secs: 1733882915 + nsecs: 967189073 +frame_id: '' and grasped_object_mask header seq: 37328 +stamp: + secs: 1733882915 + nsecs: 743684769 +frame_id: "camera_color_optical_frame" +[INFO] [1733882916.165980]: received mask_obs with header seq: 379 +stamp: + secs: 1733882916 + nsecs: 166287660 +frame_id: '' and grasped_object_mask header seq: 37334 +stamp: + secs: 1733882915 + nsecs: 943297863 +frame_id: "camera_color_optical_frame" +[INFO] [1733882916.364046]: received mask_obs with header seq: 380 +stamp: + secs: 1733882916 + nsecs: 364957809 +frame_id: '' and grasped_object_mask header seq: 37340 +stamp: + secs: 1733882916 + nsecs: 142899275 +frame_id: "camera_color_optical_frame" +[INFO] [1733882916.565768]: received mask_obs with header seq: 381 +stamp: + secs: 1733882916 + nsecs: 565814018 +frame_id: '' and grasped_object_mask header seq: 37346 +stamp: + secs: 1733882916 + nsecs: 342488766 +frame_id: "camera_color_optical_frame" +[INFO] [1733882916.760573]: received mask_obs with header seq: 382 +stamp: + secs: 1733882916 + nsecs: 761430501 +frame_id: '' and grasped_object_mask header seq: 37352 +stamp: + secs: 1733882916 + nsecs: 542066813 +frame_id: "camera_color_optical_frame" +[INFO] [1733882916.973905]: received mask_obs with header seq: 383 +stamp: + secs: 1733882916 + nsecs: 974594116 +frame_id: '' and grasped_object_mask header seq: 37358 +stamp: + secs: 1733882916 + nsecs: 741644382 +frame_id: "camera_color_optical_frame" +[INFO] [1733882917.164027]: received mask_obs with header seq: 384 +stamp: + secs: 1733882917 + nsecs: 164787769 +frame_id: '' and grasped_object_mask header seq: 37364 +stamp: + secs: 1733882916 + nsecs: 941267014 +frame_id: "camera_color_optical_frame" +[INFO] [1733882917.201880]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1733882917.966987]: Stored episode 1. +[INFO] [1733882917.967305]: Saving video... +[INFO] [1733882918.213160]: Video saved! +Starting episode 1 +[INFO] [1733882918.231754]: resetting environment +[INFO] [1733882918.242488]: cleared current plan +[INFO] [1733882918.242865]: moving to home +[INFO] [1733882921.045310]: reached home +[INFO] [1733882921.046132]: reset action history +[INFO] [1733882923.759818]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1733882941.199599]: resetting environment +[INFO] [1733882941.202505]: cleared current plan +[INFO] [1733882941.202672]: moving to home +[INFO] [1733882942.203462]: reached home +[INFO] [1733882942.204076]: reset action history +[INFO] [1733882944.911503]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/episode_rosbags/episode_1_2024-12-10-21-09-04.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-600897.log +started roslaunch server http://158.130.50.37:46289/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [601337] +process[print_text-5]: started with pid [601338] +process[rosbag_record-6]: started with pid [601362] +started bagging! +[INFO] [1733882945.602837]: received mask_obs with header seq: 385 +stamp: + secs: 1733882945 + nsecs: 603002071 +frame_id: '' and grasped_object_mask header seq: 38218 +stamp: + secs: 1733882945 + nsecs: 387733459 +frame_id: "camera_color_optical_frame" +[INFO] [1733882945.812689]: received mask_obs with header seq: 386 +stamp: + secs: 1733882945 + nsecs: 813541173 +frame_id: '' and grasped_object_mask header seq: 38224 +stamp: + secs: 1733882945 + nsecs: 587574720 +frame_id: "camera_color_optical_frame" +[INFO] [1733882945.991780]: received mask_obs with header seq: 387 +stamp: + secs: 1733882945 + nsecs: 992033958 +frame_id: '' and grasped_object_mask header seq: 38230 +stamp: + secs: 1733882945 + nsecs: 787415743 +frame_id: "camera_color_optical_frame" +[INFO] [1733882946.202628]: received mask_obs with header seq: 388 +stamp: + secs: 1733882946 + nsecs: 203542470 +frame_id: '' and grasped_object_mask header seq: 38236 +stamp: + secs: 1733882945 + nsecs: 987251520 +frame_id: "camera_color_optical_frame" +[INFO] [1733882946.429809]: received mask_obs with header seq: 389 +stamp: + secs: 1733882946 + nsecs: 430303335 +frame_id: '' and grasped_object_mask header seq: 38242 +stamp: + secs: 1733882946 + nsecs: 187087059 +frame_id: "camera_color_optical_frame" +[INFO] [1733882946.608189]: received mask_obs with header seq: 390 +stamp: + secs: 1733882946 + nsecs: 608636856 +frame_id: '' and grasped_object_mask header seq: 38248 +stamp: + secs: 1733882946 + nsecs: 386922598 +frame_id: "camera_color_optical_frame" +[INFO] [1733882946.804422]: received mask_obs with header seq: 391 +stamp: + secs: 1733882946 + nsecs: 805340528 +frame_id: '' and grasped_object_mask header seq: 38254 +stamp: + secs: 1733882946 + nsecs: 586759090 +frame_id: "camera_color_optical_frame" +[INFO] [1733882947.019040]: received mask_obs with header seq: 392 +stamp: + secs: 1733882947 + nsecs: 19926309 +frame_id: '' and grasped_object_mask header seq: 38260 +stamp: + secs: 1733882946 + nsecs: 786594152 +frame_id: "camera_color_optical_frame" +[INFO] [1733882947.209954]: received mask_obs with header seq: 393 +stamp: + secs: 1733882947 + nsecs: 210906982 +frame_id: '' and grasped_object_mask header seq: 38266 +stamp: + secs: 1733882946 + nsecs: 986429691 +frame_id: "camera_color_optical_frame" +[INFO] [1733882947.417130]: received mask_obs with header seq: 394 +stamp: + secs: 1733882947 + nsecs: 418021440 +frame_id: '' and grasped_object_mask header seq: 38272 +stamp: + secs: 1733882947 + nsecs: 186264992 +frame_id: "camera_color_optical_frame" +[INFO] [1733882947.617317]: received mask_obs with header seq: 395 +stamp: + secs: 1733882947 + nsecs: 617866039 +frame_id: '' and grasped_object_mask header seq: 38278 +stamp: + secs: 1733882947 + nsecs: 386100054 +frame_id: "camera_color_optical_frame" +[INFO] [1733882947.813208]: received mask_obs with header seq: 396 +stamp: + secs: 1733882947 + nsecs: 814121723 +frame_id: '' and grasped_object_mask header seq: 38284 +stamp: + secs: 1733882947 + nsecs: 585935354 +frame_id: "camera_color_optical_frame" +[INFO] [1733882948.019178]: received mask_obs with header seq: 397 +stamp: + secs: 1733882948 + nsecs: 19786357 +frame_id: '' and grasped_object_mask header seq: 38290 +stamp: + secs: 1733882947 + nsecs: 785771370 +frame_id: "camera_color_optical_frame" +[INFO] [1733882948.228754]: received mask_obs with header seq: 398 +stamp: + secs: 1733882948 + nsecs: 229689836 +frame_id: '' and grasped_object_mask header seq: 38296 +stamp: + secs: 1733882947 + nsecs: 985605478 +frame_id: "camera_color_optical_frame" +[INFO] [1733882948.416769]: received mask_obs with header seq: 399 +stamp: + secs: 1733882948 + nsecs: 417644262 +frame_id: '' and grasped_object_mask header seq: 38302 +stamp: + secs: 1733882948 + nsecs: 185439587 +frame_id: "camera_color_optical_frame" +[INFO] [1733882948.610952]: received mask_obs with header seq: 400 +stamp: + secs: 1733882948 + nsecs: 611837863 +frame_id: '' and grasped_object_mask header seq: 38308 +stamp: + secs: 1733882948 + nsecs: 385273457 +frame_id: "camera_color_optical_frame" +[INFO] [1733882948.823436]: received mask_obs with header seq: 401 +stamp: + secs: 1733882948 + nsecs: 824332714 +frame_id: '' and grasped_object_mask header seq: 38314 +stamp: + secs: 1733882948 + nsecs: 585107327 +frame_id: "camera_color_optical_frame" +[INFO] [1733882949.019233]: received mask_obs with header seq: 402 +stamp: + secs: 1733882949 + nsecs: 19895076 +frame_id: '' and grasped_object_mask header seq: 38320 +stamp: + secs: 1733882948 + nsecs: 784940958 +frame_id: "camera_color_optical_frame" +[INFO] [1733882949.211474]: received mask_obs with header seq: 403 +stamp: + secs: 1733882949 + nsecs: 212416410 +frame_id: '' and grasped_object_mask header seq: 38326 +stamp: + secs: 1733882948 + nsecs: 984781027 +frame_id: "camera_color_optical_frame" +[INFO] [1733882949.424443]: received mask_obs with header seq: 404 +stamp: + secs: 1733882949 + nsecs: 425297260 +frame_id: '' and grasped_object_mask header seq: 38332 +stamp: + secs: 1733882949 + nsecs: 184620380 +frame_id: "camera_color_optical_frame" +[INFO] [1733882949.613055]: received mask_obs with header seq: 405 +stamp: + secs: 1733882949 + nsecs: 613775968 +frame_id: '' and grasped_object_mask header seq: 38338 +stamp: + secs: 1733882949 + nsecs: 384459496 +frame_id: "camera_color_optical_frame" +[INFO] [1733882949.818352]: received mask_obs with header seq: 406 +stamp: + secs: 1733882949 + nsecs: 819262742 +frame_id: '' and grasped_object_mask header seq: 38344 +stamp: + secs: 1733882949 + nsecs: 584298849 +frame_id: "camera_color_optical_frame" +[INFO] [1733882950.020936]: received mask_obs with header seq: 407 +stamp: + secs: 1733882950 + nsecs: 21823406 +frame_id: '' and grasped_object_mask header seq: 38350 +stamp: + secs: 1733882949 + nsecs: 784138203 +frame_id: "camera_color_optical_frame" +[INFO] [1733882950.223955]: received mask_obs with header seq: 408 +stamp: + secs: 1733882950 + nsecs: 224575519 +frame_id: '' and grasped_object_mask header seq: 38356 +stamp: + secs: 1733882949 + nsecs: 983969927 +frame_id: "camera_color_optical_frame" +[INFO] [1733882950.424109]: received mask_obs with header seq: 409 +stamp: + secs: 1733882950 + nsecs: 424906492 +frame_id: '' and grasped_object_mask header seq: 38362 +stamp: + secs: 1733882950 + nsecs: 183801651 +frame_id: "camera_color_optical_frame" +[INFO] [1733882950.635832]: received mask_obs with header seq: 410 +stamp: + secs: 1733882950 + nsecs: 636524438 +frame_id: '' and grasped_object_mask header seq: 38368 +stamp: + secs: 1733882950 + nsecs: 383634090 +frame_id: "camera_color_optical_frame" +[INFO] [1733882950.824851]: received mask_obs with header seq: 411 +stamp: + secs: 1733882950 + nsecs: 824923276 +frame_id: '' and grasped_object_mask header seq: 38374 +stamp: + secs: 1733882950 + nsecs: 583465099 +frame_id: "camera_color_optical_frame" +[INFO] [1733882951.019611]: received mask_obs with header seq: 412 +stamp: + secs: 1733882951 + nsecs: 20526409 +frame_id: '' and grasped_object_mask header seq: 38380 +stamp: + secs: 1733882950 + nsecs: 783295870 +frame_id: "camera_color_optical_frame" +[INFO] [1733882951.222732]: received mask_obs with header seq: 413 +stamp: + secs: 1733882951 + nsecs: 223538160 +frame_id: '' and grasped_object_mask header seq: 38386 +stamp: + secs: 1733882950 + nsecs: 983139515 +frame_id: "camera_color_optical_frame" +[INFO] [1733882951.419127]: received mask_obs with header seq: 414 +stamp: + secs: 1733882951 + nsecs: 419630289 +frame_id: '' and grasped_object_mask header seq: 38392 +stamp: + secs: 1733882951 + nsecs: 182983398 +frame_id: "camera_color_optical_frame" +[INFO] [1733882951.610720]: received mask_obs with header seq: 415 +stamp: + secs: 1733882951 + nsecs: 611176252 +frame_id: '' and grasped_object_mask header seq: 38398 +stamp: + secs: 1733882951 + nsecs: 382827282 +frame_id: "camera_color_optical_frame" +[INFO] [1733882951.811968]: received mask_obs with header seq: 416 +stamp: + secs: 1733882951 + nsecs: 812586545 +frame_id: '' and grasped_object_mask header seq: 38404 +stamp: + secs: 1733882951 + nsecs: 582672834 +frame_id: "camera_color_optical_frame" +[INFO] [1733882952.019268]: received mask_obs with header seq: 417 +stamp: + secs: 1733882952 + nsecs: 20040512 +frame_id: '' and grasped_object_mask header seq: 38410 +stamp: + secs: 1733882951 + nsecs: 782517433 +frame_id: "camera_color_optical_frame" +[INFO] [1733882952.222058]: received mask_obs with header seq: 418 +stamp: + secs: 1733882952 + nsecs: 222842931 +frame_id: '' and grasped_object_mask header seq: 38416 +stamp: + secs: 1733882951 + nsecs: 982357502 +frame_id: "camera_color_optical_frame" +[INFO] [1733882952.420488]: received mask_obs with header seq: 419 +stamp: + secs: 1733882952 + nsecs: 420991659 +frame_id: '' and grasped_object_mask header seq: 38422 +stamp: + secs: 1733882952 + nsecs: 182197332 +frame_id: "camera_color_optical_frame" +[INFO] [1733882952.619056]: received mask_obs with header seq: 420 +stamp: + secs: 1733882952 + nsecs: 619902610 +frame_id: '' and grasped_object_mask header seq: 38428 +stamp: + secs: 1733882952 + nsecs: 382037640 +frame_id: "camera_color_optical_frame" +[INFO] [1733882952.810164]: received mask_obs with header seq: 421 +stamp: + secs: 1733882952 + nsecs: 811100959 +frame_id: '' and grasped_object_mask header seq: 38434 +stamp: + secs: 1733882952 + nsecs: 581877708 +frame_id: "camera_color_optical_frame" +[INFO] [1733882953.020753]: received mask_obs with header seq: 422 +stamp: + secs: 1733882953 + nsecs: 21631002 +frame_id: '' and grasped_object_mask header seq: 38440 +stamp: + secs: 1733882952 + nsecs: 781718969 +frame_id: "camera_color_optical_frame" +[INFO] [1733882953.220563]: received mask_obs with header seq: 423 +stamp: + secs: 1733882953 + nsecs: 221395015 +frame_id: '' and grasped_object_mask header seq: 38446 +stamp: + secs: 1733882952 + nsecs: 981560707 +frame_id: "camera_color_optical_frame" +[INFO] [1733882953.420057]: received mask_obs with header seq: 424 +stamp: + secs: 1733882953 + nsecs: 420937776 +frame_id: '' and grasped_object_mask header seq: 38452 +stamp: + secs: 1733882953 + nsecs: 181403160 +frame_id: "camera_color_optical_frame" +[INFO] [1733882953.614997]: received mask_obs with header seq: 425 +stamp: + secs: 1733882953 + nsecs: 615551471 +frame_id: '' and grasped_object_mask header seq: 38458 +stamp: + secs: 1733882953 + nsecs: 381245375 +frame_id: "camera_color_optical_frame" +[INFO] [1733882953.819748]: received mask_obs with header seq: 426 +stamp: + secs: 1733882953 + nsecs: 820515871 +frame_id: '' and grasped_object_mask header seq: 38464 +stamp: + secs: 1733882953 + nsecs: 581087589 +frame_id: "camera_color_optical_frame" +[INFO] [1733882954.006769]: received mask_obs with header seq: 427 +stamp: + secs: 1733882954 + nsecs: 7503032 +frame_id: '' and grasped_object_mask header seq: 38470 +stamp: + secs: 1733882953 + nsecs: 780930519 +frame_id: "camera_color_optical_frame" +[INFO] [1733882954.221330]: received mask_obs with header seq: 428 +stamp: + secs: 1733882954 + nsecs: 222125768 +frame_id: '' and grasped_object_mask header seq: 38476 +stamp: + secs: 1733882953 + nsecs: 980767727 +frame_id: "camera_color_optical_frame" +[INFO] [1733882954.434056]: received mask_obs with header seq: 429 +stamp: + secs: 1733882954 + nsecs: 434839725 +frame_id: '' and grasped_object_mask header seq: 38482 +stamp: + secs: 1733882954 + nsecs: 180603504 +frame_id: "camera_color_optical_frame" +[INFO] [1733882954.629796]: received mask_obs with header seq: 430 +stamp: + secs: 1733882954 + nsecs: 630671024 +frame_id: '' and grasped_object_mask header seq: 38488 +stamp: + secs: 1733882954 + nsecs: 380439520 +frame_id: "camera_color_optical_frame" +[INFO] [1733882954.829654]: received mask_obs with header seq: 431 +stamp: + secs: 1733882954 + nsecs: 830207347 +frame_id: '' and grasped_object_mask header seq: 38494 +stamp: + secs: 1733882954 + nsecs: 580275059 +frame_id: "camera_color_optical_frame" +[INFO] [1733882955.013008]: received mask_obs with header seq: 432 +stamp: + secs: 1733882955 + nsecs: 13230323 +frame_id: '' and grasped_object_mask header seq: 38500 +stamp: + secs: 1733882954 + nsecs: 780110598 +frame_id: "camera_color_optical_frame" +[INFO] [1733882955.214613]: received mask_obs with header seq: 433 +stamp: + secs: 1733882955 + nsecs: 215448141 +frame_id: '' and grasped_object_mask header seq: 38506 +stamp: + secs: 1733882954 + nsecs: 979944468 +frame_id: "camera_color_optical_frame" +[INFO] [1733882955.424921]: received mask_obs with header seq: 434 +stamp: + secs: 1733882955 + nsecs: 425194025 +frame_id: '' and grasped_object_mask header seq: 38512 +stamp: + secs: 1733882955 + nsecs: 179779053 +frame_id: "camera_color_optical_frame" +[INFO] [1733882955.618148]: received mask_obs with header seq: 435 +stamp: + secs: 1733882955 + nsecs: 618857145 +frame_id: '' and grasped_object_mask header seq: 38518 +stamp: + secs: 1733882955 + nsecs: 379612207 +frame_id: "camera_color_optical_frame" +[INFO] [1733882955.819377]: received mask_obs with header seq: 436 +stamp: + secs: 1733882955 + nsecs: 819941282 +frame_id: '' and grasped_object_mask header seq: 38524 +stamp: + secs: 1733882955 + nsecs: 579445601 +frame_id: "camera_color_optical_frame" +[INFO] [1733882956.010765]: received mask_obs with header seq: 437 +stamp: + secs: 1733882956 + nsecs: 11482954 +frame_id: '' and grasped_object_mask header seq: 38530 +stamp: + secs: 1733882955 + nsecs: 779278278 +frame_id: "camera_color_optical_frame" +[INFO] [1733882956.214671]: received mask_obs with header seq: 438 +stamp: + secs: 1733882956 + nsecs: 215225458 +frame_id: '' and grasped_object_mask header seq: 38536 +stamp: + secs: 1733882955 + nsecs: 979115725 +frame_id: "camera_color_optical_frame" +[INFO] [1733882956.411877]: received mask_obs with header seq: 439 +stamp: + secs: 1733882956 + nsecs: 412673234 +frame_id: '' and grasped_object_mask header seq: 38542 +stamp: + secs: 1733882956 + nsecs: 178953409 +frame_id: "camera_color_optical_frame" +[INFO] [1733882956.620041]: received mask_obs with header seq: 440 +stamp: + secs: 1733882956 + nsecs: 620893001 +frame_id: '' and grasped_object_mask header seq: 38548 +stamp: + secs: 1733882956 + nsecs: 378792048 +frame_id: "camera_color_optical_frame" +[INFO] [1733882956.820601]: received mask_obs with header seq: 441 +stamp: + secs: 1733882956 + nsecs: 821280956 +frame_id: '' and grasped_object_mask header seq: 38554 +stamp: + secs: 1733882956 + nsecs: 578629732 +frame_id: "camera_color_optical_frame" +[INFO] [1733882957.014343]: received mask_obs with header seq: 442 +stamp: + secs: 1733882957 + nsecs: 14899492 +frame_id: '' and grasped_object_mask header seq: 38560 +stamp: + secs: 1733882956 + nsecs: 778467178 +frame_id: "camera_color_optical_frame" +[INFO] [1733882957.206022]: received mask_obs with header seq: 443 +stamp: + secs: 1733882957 + nsecs: 206637382 +frame_id: '' and grasped_object_mask header seq: 38566 +stamp: + secs: 1733882956 + nsecs: 978302956 +frame_id: "camera_color_optical_frame" +[INFO] [1733882957.409874]: received mask_obs with header seq: 444 +stamp: + secs: 1733882957 + nsecs: 410565376 +frame_id: '' and grasped_object_mask header seq: 38572 +stamp: + secs: 1733882957 + nsecs: 178138733 +frame_id: "camera_color_optical_frame" +[INFO] [1733882957.608946]: received mask_obs with header seq: 445 +stamp: + secs: 1733882957 + nsecs: 609660387 +frame_id: '' and grasped_object_mask header seq: 38578 +stamp: + secs: 1733882957 + nsecs: 377974272 +frame_id: "camera_color_optical_frame" +[INFO] [1733882957.813575]: received mask_obs with header seq: 446 +stamp: + secs: 1733882957 + nsecs: 814331531 +frame_id: '' and grasped_object_mask header seq: 38584 +stamp: + secs: 1733882957 + nsecs: 577810049 +frame_id: "camera_color_optical_frame" +[INFO] [1733882958.024134]: received mask_obs with header seq: 447 +stamp: + secs: 1733882958 + nsecs: 24856090 +frame_id: '' and grasped_object_mask header seq: 38590 +stamp: + secs: 1733882957 + nsecs: 777647018 +frame_id: "camera_color_optical_frame" +[INFO] [1733882958.235514]: received mask_obs with header seq: 448 +stamp: + secs: 1733882958 + nsecs: 236265897 +frame_id: '' and grasped_object_mask header seq: 38596 +stamp: + secs: 1733882957 + nsecs: 977482796 +frame_id: "camera_color_optical_frame" +[INFO] [1733882958.442382]: received mask_obs with header seq: 449 +stamp: + secs: 1733882958 + nsecs: 443145036 +frame_id: '' and grasped_object_mask header seq: 38603 +stamp: + secs: 1733882958 + nsecs: 210625410 +frame_id: "camera_color_optical_frame" +[INFO] [1733882958.632716]: received mask_obs with header seq: 450 +stamp: + secs: 1733882958 + nsecs: 633198022 +frame_id: '' and grasped_object_mask header seq: 38608 +stamp: + secs: 1733882958 + nsecs: 377154827 +frame_id: "camera_color_optical_frame" +[INFO] [1733882958.820298]: received mask_obs with header seq: 451 +stamp: + secs: 1733882958 + nsecs: 821182012 +frame_id: '' and grasped_object_mask header seq: 38614 +stamp: + secs: 1733882958 + nsecs: 576990128 +frame_id: "camera_color_optical_frame" +[INFO] [1733882959.010772]: received mask_obs with header seq: 452 +stamp: + secs: 1733882959 + nsecs: 11446475 +frame_id: '' and grasped_object_mask header seq: 38620 +stamp: + secs: 1733882958 + nsecs: 776825905 +frame_id: "camera_color_optical_frame" +[INFO] [1733882959.210433]: received mask_obs with header seq: 453 +stamp: + secs: 1733882959 + nsecs: 211297512 +frame_id: '' and grasped_object_mask header seq: 38626 +stamp: + secs: 1733882958 + nsecs: 976668119 +frame_id: "camera_color_optical_frame" +[INFO] [1733882959.410849]: received mask_obs with header seq: 454 +stamp: + secs: 1733882959 + nsecs: 411695718 +frame_id: '' and grasped_object_mask header seq: 38632 +stamp: + secs: 1733882959 + nsecs: 176510334 +frame_id: "camera_color_optical_frame" +[INFO] [1733882959.618658]: received mask_obs with header seq: 455 +stamp: + secs: 1733882959 + nsecs: 619433164 +frame_id: '' and grasped_object_mask header seq: 38638 +stamp: + secs: 1733882959 + nsecs: 376352549 +frame_id: "camera_color_optical_frame" +[INFO] [1733882959.818611]: received mask_obs with header seq: 456 +stamp: + secs: 1733882959 + nsecs: 819406032 +frame_id: '' and grasped_object_mask header seq: 38644 +stamp: + secs: 1733882959 + nsecs: 576195002 +frame_id: "camera_color_optical_frame" +[INFO] [1733882960.008700]: received mask_obs with header seq: 457 +stamp: + secs: 1733882960 + nsecs: 9556055 +frame_id: '' and grasped_object_mask header seq: 38650 +stamp: + secs: 1733882959 + nsecs: 776037693 +frame_id: "camera_color_optical_frame" +[INFO] [1733882960.217906]: received mask_obs with header seq: 458 +stamp: + secs: 1733882960 + nsecs: 218761205 +frame_id: '' and grasped_object_mask header seq: 38656 +stamp: + secs: 1733882959 + nsecs: 975876093 +frame_id: "camera_color_optical_frame" +[INFO] [1733882960.416972]: received mask_obs with header seq: 459 +stamp: + secs: 1733882960 + nsecs: 417704105 +frame_id: '' and grasped_object_mask header seq: 38662 +stamp: + secs: 1733882960 + nsecs: 175714731 +frame_id: "camera_color_optical_frame" +[INFO] [1733882960.616354]: received mask_obs with header seq: 460 +stamp: + secs: 1733882960 + nsecs: 617254495 +frame_id: '' and grasped_object_mask header seq: 38668 +stamp: + secs: 1733882960 + nsecs: 375552654 +frame_id: "camera_color_optical_frame" +[INFO] [1733882960.817847]: received mask_obs with header seq: 461 +stamp: + secs: 1733882960 + nsecs: 818738460 +frame_id: '' and grasped_object_mask header seq: 38674 +stamp: + secs: 1733882960 + nsecs: 575390577 +frame_id: "camera_color_optical_frame" +[INFO] [1733882961.018188]: received mask_obs with header seq: 462 +stamp: + secs: 1733882961 + nsecs: 19055366 +frame_id: '' and grasped_object_mask header seq: 38680 +stamp: + secs: 1733882960 + nsecs: 775228739 +frame_id: "camera_color_optical_frame" +[INFO] [1733882961.210039]: received mask_obs with header seq: 463 +stamp: + secs: 1733882961 + nsecs: 210726976 +frame_id: '' and grasped_object_mask header seq: 38686 +stamp: + secs: 1733882960 + nsecs: 976572990 +frame_id: "camera_color_optical_frame" +[INFO] [1733882961.410115]: received mask_obs with header seq: 464 +stamp: + secs: 1733882961 + nsecs: 410964488 +frame_id: '' and grasped_object_mask header seq: 38692 +stamp: + secs: 1733882961 + nsecs: 178742409 +frame_id: "camera_color_optical_frame" +[INFO] [1733882961.608350]: received mask_obs with header seq: 465 +stamp: + secs: 1733882961 + nsecs: 608841896 +frame_id: '' and grasped_object_mask header seq: 38698 +stamp: + secs: 1733882961 + nsecs: 380999088 +frame_id: "camera_color_optical_frame" +[INFO] [1733882961.816840]: received mask_obs with header seq: 466 +stamp: + secs: 1733882961 + nsecs: 817731380 +frame_id: '' and grasped_object_mask header seq: 38704 +stamp: + secs: 1733882961 + nsecs: 583341122 +frame_id: "camera_color_optical_frame" +[INFO] [1733882962.017221]: received mask_obs with header seq: 467 +stamp: + secs: 1733882962 + nsecs: 18062114 +frame_id: '' and grasped_object_mask header seq: 38710 +stamp: + secs: 1733882961 + nsecs: 785769224 +frame_id: "camera_color_optical_frame" +[INFO] [1733882962.223751]: received mask_obs with header seq: 468 +stamp: + secs: 1733882962 + nsecs: 224606037 +frame_id: '' and grasped_object_mask header seq: 38716 +stamp: + secs: 1733882961 + nsecs: 986351252 +frame_id: "camera_color_optical_frame" +[INFO] [1733882962.424268]: received mask_obs with header seq: 469 +stamp: + secs: 1733882962 + nsecs: 425185441 +frame_id: '' and grasped_object_mask header seq: 38722 +stamp: + secs: 1733882962 + nsecs: 185972452 +frame_id: "camera_color_optical_frame" +[INFO] [1733882962.615367]: received mask_obs with header seq: 470 +stamp: + secs: 1733882962 + nsecs: 616153001 +frame_id: '' and grasped_object_mask header seq: 38728 +stamp: + secs: 1733882962 + nsecs: 385582447 +frame_id: "camera_color_optical_frame" +[INFO] [1733882962.816738]: received mask_obs with header seq: 471 +stamp: + secs: 1733882962 + nsecs: 817471742 +frame_id: '' and grasped_object_mask header seq: 38734 +stamp: + secs: 1733882962 + nsecs: 585182667 +frame_id: "camera_color_optical_frame" +[INFO] [1733882963.030600]: received mask_obs with header seq: 472 +stamp: + secs: 1733882963 + nsecs: 31467437 +frame_id: '' and grasped_object_mask header seq: 38740 +stamp: + secs: 1733882962 + nsecs: 784770489 +frame_id: "camera_color_optical_frame" +[INFO] [1733882963.227490]: received mask_obs with header seq: 473 +stamp: + secs: 1733882963 + nsecs: 228056669 +frame_id: '' and grasped_object_mask header seq: 38746 +stamp: + secs: 1733882962 + nsecs: 984370470 +frame_id: "camera_color_optical_frame" +[INFO] [1733882963.417822]: received mask_obs with header seq: 474 +stamp: + secs: 1733882963 + nsecs: 418559551 +frame_id: '' and grasped_object_mask header seq: 38752 +stamp: + secs: 1733882963 + nsecs: 183969975 +frame_id: "camera_color_optical_frame" +[INFO] [1733882963.619098]: received mask_obs with header seq: 475 +stamp: + secs: 1733882963 + nsecs: 619916915 +frame_id: '' and grasped_object_mask header seq: 38758 +stamp: + secs: 1733882963 + nsecs: 383557558 +frame_id: "camera_color_optical_frame" +[INFO] [1733882963.813482]: received mask_obs with header seq: 476 +stamp: + secs: 1733882963 + nsecs: 814132690 +frame_id: '' and grasped_object_mask header seq: 38764 +stamp: + secs: 1733882963 + nsecs: 583133459 +frame_id: "camera_color_optical_frame" +[INFO] [1733882964.017485]: received mask_obs with header seq: 477 +stamp: + secs: 1733882964 + nsecs: 18350839 +frame_id: '' and grasped_object_mask header seq: 38770 +stamp: + secs: 1733882963 + nsecs: 782698393 +frame_id: "camera_color_optical_frame" +[INFO] [1733882964.215408]: received mask_obs with header seq: 478 +stamp: + secs: 1733882964 + nsecs: 216079711 +frame_id: '' and grasped_object_mask header seq: 38777 +stamp: + secs: 1733882964 + nsecs: 15566111 +frame_id: "camera_color_optical_frame" +[INFO] [1733882964.416433]: received mask_obs with header seq: 479 +stamp: + secs: 1733882964 + nsecs: 417154073 +frame_id: '' and grasped_object_mask header seq: 38782 +stamp: + secs: 1733882964 + nsecs: 181908131 +frame_id: "camera_color_optical_frame" +[INFO] [1733882964.611886]: received mask_obs with header seq: 480 +stamp: + secs: 1733882964 + nsecs: 612670660 +frame_id: '' and grasped_object_mask header seq: 38788 +stamp: + secs: 1733882964 + nsecs: 381507397 +frame_id: "camera_color_optical_frame" +[INFO] [1733882964.808587]: received mask_obs with header seq: 481 +stamp: + secs: 1733882964 + nsecs: 809448957 +frame_id: '' and grasped_object_mask header seq: 38794 +stamp: + secs: 1733882964 + nsecs: 581095219 +frame_id: "camera_color_optical_frame" +[INFO] [1733882965.015636]: received mask_obs with header seq: 482 +stamp: + secs: 1733882965 + nsecs: 16499519 +frame_id: '' and grasped_object_mask header seq: 38800 +stamp: + secs: 1733882964 + nsecs: 780671358 +frame_id: "camera_color_optical_frame" +[INFO] [1733882965.217267]: received mask_obs with header seq: 483 +stamp: + secs: 1733882965 + nsecs: 218077182 +frame_id: '' and grasped_object_mask header seq: 38806 +stamp: + secs: 1733882964 + nsecs: 980354786 +frame_id: "camera_color_optical_frame" +[INFO] [1733882965.430651]: received mask_obs with header seq: 484 +stamp: + secs: 1733882965 + nsecs: 431488275 +frame_id: '' and grasped_object_mask header seq: 38812 +stamp: + secs: 1733882965 + nsecs: 179960966 +frame_id: "camera_color_optical_frame" +[INFO] [1733882965.622650]: received mask_obs with header seq: 485 +stamp: + secs: 1733882965 + nsecs: 623339891 +frame_id: '' and grasped_object_mask header seq: 38818 +stamp: + secs: 1733882965 + nsecs: 379554749 +frame_id: "camera_color_optical_frame" +[INFO] [1733882965.821792]: received mask_obs with header seq: 486 +stamp: + secs: 1733882965 + nsecs: 822372674 +frame_id: '' and grasped_object_mask header seq: 38824 +stamp: + secs: 1733882965 + nsecs: 579136610 +frame_id: "camera_color_optical_frame" +[INFO] [1733882966.021492]: received mask_obs with header seq: 487 +stamp: + secs: 1733882966 + nsecs: 21777868 +frame_id: '' and grasped_object_mask header seq: 38831 +stamp: + secs: 1733882965 + nsecs: 811968088 +frame_id: "camera_color_optical_frame" +[INFO] [1733882966.219381]: received mask_obs with header seq: 488 +stamp: + secs: 1733882966 + nsecs: 219886779 +frame_id: '' and grasped_object_mask header seq: 38836 +stamp: + secs: 1733882965 + nsecs: 978288889 +frame_id: "camera_color_optical_frame" +[INFO] [1733882966.425884]: received mask_obs with header seq: 489 +stamp: + secs: 1733882966 + nsecs: 425769805 +frame_id: '' and grasped_object_mask header seq: 38843 +stamp: + secs: 1733882966 + nsecs: 211161375 +frame_id: "camera_color_optical_frame" +[INFO] [1733882966.616571]: received mask_obs with header seq: 490 +stamp: + secs: 1733882966 + nsecs: 617453813 +frame_id: '' and grasped_object_mask header seq: 38849 +stamp: + secs: 1733882966 + nsecs: 410753250 +frame_id: "camera_color_optical_frame" +[INFO] [1733882966.825200]: received mask_obs with header seq: 491 +stamp: + secs: 1733882966 + nsecs: 825361728 +frame_id: '' and grasped_object_mask header seq: 38855 +stamp: + secs: 1733882966 + nsecs: 610333204 +frame_id: "camera_color_optical_frame" +[INFO] [1733882967.028847]: received mask_obs with header seq: 492 +stamp: + secs: 1733882967 + nsecs: 29453754 +frame_id: '' and grasped_object_mask header seq: 38861 +stamp: + secs: 1733882966 + nsecs: 809901476 +frame_id: "camera_color_optical_frame" +[INFO] [1733882967.222150]: received mask_obs with header seq: 493 +stamp: + secs: 1733882967 + nsecs: 222902774 +frame_id: '' and grasped_object_mask header seq: 38867 +stamp: + secs: 1733882967 + nsecs: 9498119 +frame_id: "camera_color_optical_frame" +[INFO] [1733882967.421842]: received mask_obs with header seq: 494 +stamp: + secs: 1733882967 + nsecs: 422706365 +frame_id: '' and grasped_object_mask header seq: 38873 +stamp: + secs: 1733882967 + nsecs: 209113121 +frame_id: "camera_color_optical_frame" +[INFO] [1733882967.625467]: received mask_obs with header seq: 495 +stamp: + secs: 1733882967 + nsecs: 626407384 +frame_id: '' and grasped_object_mask header seq: 38879 +stamp: + secs: 1733882967 + nsecs: 408716679 +frame_id: "camera_color_optical_frame" +[INFO] [1733882967.823421]: received mask_obs with header seq: 496 +stamp: + secs: 1733882967 + nsecs: 824296474 +frame_id: '' and grasped_object_mask header seq: 38885 +stamp: + secs: 1733882967 + nsecs: 608309507 +frame_id: "camera_color_optical_frame" +[INFO] [1733882968.020759]: received mask_obs with header seq: 497 +stamp: + secs: 1733882968 + nsecs: 20812273 +frame_id: '' and grasped_object_mask header seq: 38891 +stamp: + secs: 1733882967 + nsecs: 807891130 +frame_id: "camera_color_optical_frame" +[INFO] [1733882968.238090]: received mask_obs with header seq: 498 +stamp: + secs: 1733882968 + nsecs: 238969564 +frame_id: '' and grasped_object_mask header seq: 38897 +stamp: + secs: 1733882968 + nsecs: 7488251 +frame_id: "camera_color_optical_frame" +[INFO] [1733882968.429459]: received mask_obs with header seq: 499 +stamp: + secs: 1733882968 + nsecs: 430408239 +frame_id: '' and grasped_object_mask header seq: 38903 +stamp: + secs: 1733882968 + nsecs: 207095861 +frame_id: "camera_color_optical_frame" +[INFO] [1733882968.638610]: received mask_obs with header seq: 500 +stamp: + secs: 1733882968 + nsecs: 639400243 +frame_id: '' and grasped_object_mask header seq: 38909 +stamp: + secs: 1733882968 + nsecs: 406691313 +frame_id: "camera_color_optical_frame" +[INFO] [1733882968.834359]: received mask_obs with header seq: 501 +stamp: + secs: 1733882968 + nsecs: 835080862 +frame_id: '' and grasped_object_mask header seq: 38915 +stamp: + secs: 1733882968 + nsecs: 606275082 +frame_id: "camera_color_optical_frame" +[INFO] [1733882969.038820]: received mask_obs with header seq: 502 +stamp: + secs: 1733882969 + nsecs: 39704084 +frame_id: '' and grasped_object_mask header seq: 38921 +stamp: + secs: 1733882968 + nsecs: 805847406 +frame_id: "camera_color_optical_frame" +[INFO] [1733882969.260651]: received mask_obs with header seq: 503 +stamp: + secs: 1733882969 + nsecs: 261457681 +frame_id: '' and grasped_object_mask header seq: 38927 +stamp: + secs: 1733882969 + nsecs: 5445242 +frame_id: "camera_color_optical_frame" +[INFO] [1733882969.461192]: received mask_obs with header seq: 504 +stamp: + secs: 1733882969 + nsecs: 461972951 +frame_id: '' and grasped_object_mask header seq: 38934 +stamp: + secs: 1733882969 + nsecs: 238330841 +frame_id: "camera_color_optical_frame" +[INFO] [1733882969.656621]: received mask_obs with header seq: 505 +stamp: + secs: 1733882969 + nsecs: 657253026 +frame_id: '' and grasped_object_mask header seq: 38939 +stamp: + secs: 1733882969 + nsecs: 404668570 +frame_id: "camera_color_optical_frame" +[INFO] [1733882969.838645]: received mask_obs with header seq: 506 +stamp: + secs: 1733882969 + nsecs: 839253425 +frame_id: '' and grasped_object_mask header seq: 38945 +stamp: + secs: 1733882969 + nsecs: 604262590 +frame_id: "camera_color_optical_frame" +[INFO] [1733882970.032465]: received mask_obs with header seq: 507 +stamp: + secs: 1733882970 + nsecs: 33417940 +frame_id: '' and grasped_object_mask header seq: 38951 +stamp: + secs: 1733882969 + nsecs: 803845882 +frame_id: "camera_color_optical_frame" +[INFO] [1733882970.231637]: received mask_obs with header seq: 508 +stamp: + secs: 1733882970 + nsecs: 232269048 +frame_id: '' and grasped_object_mask header seq: 38957 +stamp: + secs: 1733882970 + nsecs: 3447771 +frame_id: "camera_color_optical_frame" +[INFO] [1733882970.432849]: received mask_obs with header seq: 509 +stamp: + secs: 1733882970 + nsecs: 433695077 +frame_id: '' and grasped_object_mask header seq: 38963 +stamp: + secs: 1733882970 + nsecs: 203062296 +frame_id: "camera_color_optical_frame" +[INFO] [1733882970.630198]: received mask_obs with header seq: 510 +stamp: + secs: 1733882970 + nsecs: 630913496 +frame_id: '' and grasped_object_mask header seq: 38969 +stamp: + secs: 1733882970 + nsecs: 402665377 +frame_id: "camera_color_optical_frame" +[INFO] [1733882970.827176]: received mask_obs with header seq: 511 +stamp: + secs: 1733882970 + nsecs: 828046798 +frame_id: '' and grasped_object_mask header seq: 38975 +stamp: + secs: 1733882970 + nsecs: 602257967 +frame_id: "camera_color_optical_frame" +[INFO] [1733882971.022573]: received mask_obs with header seq: 512 +stamp: + secs: 1733882971 + nsecs: 23345470 +frame_id: '' and grasped_object_mask header seq: 38981 +stamp: + secs: 1733882970 + nsecs: 801838398 +frame_id: "camera_color_optical_frame" +[INFO] [1733882971.224832]: received mask_obs with header seq: 513 +stamp: + secs: 1733882971 + nsecs: 225498914 +frame_id: '' and grasped_object_mask header seq: 38987 +stamp: + secs: 1733882971 + nsecs: 1437664 +frame_id: "camera_color_optical_frame" +[INFO] [1733882971.434679]: received mask_obs with header seq: 514 +stamp: + secs: 1733882971 + nsecs: 435533285 +frame_id: '' and grasped_object_mask header seq: 38993 +stamp: + secs: 1733882971 + nsecs: 201050758 +frame_id: "camera_color_optical_frame" +[INFO] [1733882971.630450]: received mask_obs with header seq: 515 +stamp: + secs: 1733882971 + nsecs: 631337165 +frame_id: '' and grasped_object_mask header seq: 38999 +stamp: + secs: 1733882971 + nsecs: 400652170 +frame_id: "camera_color_optical_frame" +[INFO] [1733882971.824710]: received mask_obs with header seq: 516 +stamp: + secs: 1733882971 + nsecs: 825038433 +frame_id: '' and grasped_object_mask header seq: 39005 +stamp: + secs: 1733882971 + nsecs: 600241899 +frame_id: "camera_color_optical_frame" +[INFO] [1733882972.015732]: received mask_obs with header seq: 517 +stamp: + secs: 1733882972 + nsecs: 16603469 +frame_id: '' and grasped_object_mask header seq: 39011 +stamp: + secs: 1733882971 + nsecs: 799821615 +frame_id: "camera_color_optical_frame" +[INFO] [1733882972.219994]: received mask_obs with header seq: 518 +stamp: + secs: 1733882972 + nsecs: 220815181 +frame_id: '' and grasped_object_mask header seq: 39017 +stamp: + secs: 1733882971 + nsecs: 999419928 +frame_id: "camera_color_optical_frame" +[INFO] [1733882972.419331]: received mask_obs with header seq: 519 +stamp: + secs: 1733882972 + nsecs: 420067310 +frame_id: '' and grasped_object_mask header seq: 39023 +stamp: + secs: 1733882972 + nsecs: 199033022 +frame_id: "camera_color_optical_frame" +[INFO] [1733882972.449565]: Storing episode... +[print_text-5] killing on exit[pub_text-4] killing on exit +[rosbag_record-6] killing on exit +[INFO] [1733882973.250117]: Stored episode 2. +[INFO] [1733882973.250624]: Saving video... +[INFO] [1733882973.570604]: Video saved! +Starting episode 2 +[INFO] [1733882973.600720]: resetting environment +[INFO] [1733882973.611490]: cleared current plan +[INFO] [1733882973.611801]: moving to home +[INFO] [1733882976.418311]: reached home +[INFO] [1733882976.421374]: reset action history +[INFO] [1733882979.147022]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1733882981.454751]: resetting environment +[INFO] [1733882981.467842]: cleared current plan +[INFO] [1733882981.468148]: moving to home +[INFO] [1733882982.469825]: reached home +[INFO] [1733882982.470292]: reset action history +[INFO] [1733882985.181373]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/episode_rosbags/episode_2_2024-12-10-21-09-45.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-600897.log +started roslaunch server http://158.130.50.37:40793/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [601497] +process[print_text-8]: started with pid [601498] +process[rosbag_record-9]: started with pid [601499] +started bagging! +[INFO] [1733882986.013583]: received mask_obs with header seq: 520 +stamp: + secs: 1733882986 + nsecs: 14185667 +frame_id: '' and grasped_object_mask header seq: 39432 +stamp: + secs: 1733882985 + nsecs: 821710348 +frame_id: "camera_color_optical_frame" +[INFO] [1733882986.221640]: received mask_obs with header seq: 521 +stamp: + secs: 1733882986 + nsecs: 222427129 +frame_id: '' and grasped_object_mask header seq: 39438 +stamp: + secs: 1733882986 + nsecs: 21548271 +frame_id: "camera_color_optical_frame" +[INFO] [1733882986.425108]: received mask_obs with header seq: 522 +stamp: + secs: 1733882986 + nsecs: 425947904 +frame_id: '' and grasped_object_mask header seq: 39444 +stamp: + secs: 1733882986 + nsecs: 221383333 +frame_id: "camera_color_optical_frame" +[INFO] [1733882986.625793]: received mask_obs with header seq: 523 +stamp: + secs: 1733882986 + nsecs: 626699686 +frame_id: '' and grasped_object_mask header seq: 39450 +stamp: + secs: 1733882986 + nsecs: 421218157 +frame_id: "camera_color_optical_frame" +[INFO] [1733882986.836624]: received mask_obs with header seq: 524 +stamp: + secs: 1733882986 + nsecs: 835594177 +frame_id: '' and grasped_object_mask header seq: 39456 +stamp: + secs: 1733882986 + nsecs: 621053219 +frame_id: "camera_color_optical_frame" +[INFO] [1733882987.032621]: received mask_obs with header seq: 525 +stamp: + secs: 1733882987 + nsecs: 33390283 +frame_id: '' and grasped_object_mask header seq: 39462 +stamp: + secs: 1733882986 + nsecs: 820888042 +frame_id: "camera_color_optical_frame" +[INFO] [1733882987.248609]: received mask_obs with header seq: 526 +stamp: + secs: 1733882987 + nsecs: 249329805 +frame_id: '' and grasped_object_mask header seq: 39468 +stamp: + secs: 1733882987 + nsecs: 20729303 +frame_id: "camera_color_optical_frame" +[INFO] [1733882987.452894]: received mask_obs with header seq: 527 +stamp: + secs: 1733882987 + nsecs: 453576564 +frame_id: '' and grasped_object_mask header seq: 39474 +stamp: + secs: 1733882987 + nsecs: 220574856 +frame_id: "camera_color_optical_frame" +[INFO] [1733882987.647550]: received mask_obs with header seq: 528 +stamp: + secs: 1733882987 + nsecs: 648237705 +frame_id: '' and grasped_object_mask header seq: 39480 +stamp: + secs: 1733882987 + nsecs: 420419931 +frame_id: "camera_color_optical_frame" +[INFO] [1733882987.844383]: received mask_obs with header seq: 529 +stamp: + secs: 1733882987 + nsecs: 844470977 +frame_id: '' and grasped_object_mask header seq: 39486 +stamp: + secs: 1733882987 + nsecs: 620265245 +frame_id: "camera_color_optical_frame" +[INFO] [1733882988.048652]: received mask_obs with header seq: 530 +stamp: + secs: 1733882988 + nsecs: 49498319 +frame_id: '' and grasped_object_mask header seq: 39492 +stamp: + secs: 1733882987 + nsecs: 820110798 +frame_id: "camera_color_optical_frame" +[INFO] [1733882988.236767]: received mask_obs with header seq: 531 +stamp: + secs: 1733882988 + nsecs: 237669944 +frame_id: '' and grasped_object_mask header seq: 39498 +stamp: + secs: 1733882988 + nsecs: 19951582 +frame_id: "camera_color_optical_frame" +[INFO] [1733882988.446868]: received mask_obs with header seq: 532 +stamp: + secs: 1733882988 + nsecs: 447345972 +frame_id: '' and grasped_object_mask header seq: 39504 +stamp: + secs: 1733882988 + nsecs: 219789743 +frame_id: "camera_color_optical_frame" +[INFO] [1733882988.650209]: received mask_obs with header seq: 533 +stamp: + secs: 1733882988 + nsecs: 650863647 +frame_id: '' and grasped_object_mask header seq: 39510 +stamp: + secs: 1733882988 + nsecs: 419628143 +frame_id: "camera_color_optical_frame" +[INFO] [1733882988.856493]: received mask_obs with header seq: 534 +stamp: + secs: 1733882988 + nsecs: 857425451 +frame_id: '' and grasped_object_mask header seq: 39516 +stamp: + secs: 1733882988 + nsecs: 619467497 +frame_id: "camera_color_optical_frame" +[INFO] [1733882989.068754]: received mask_obs with header seq: 535 +stamp: + secs: 1733882989 + nsecs: 68956851 +frame_id: '' and grasped_object_mask header seq: 39522 +stamp: + secs: 1733882988 + nsecs: 819305897 +frame_id: "camera_color_optical_frame" +[INFO] [1733882989.273335]: received mask_obs with header seq: 536 +stamp: + secs: 1733882989 + nsecs: 273637771 +frame_id: '' and grasped_object_mask header seq: 39529 +stamp: + secs: 1733882989 + nsecs: 52451134 +frame_id: "camera_color_optical_frame" +[INFO] [1733882989.456315]: received mask_obs with header seq: 537 +stamp: + secs: 1733882989 + nsecs: 456853151 +frame_id: '' and grasped_object_mask header seq: 39534 +stamp: + secs: 1733882989 + nsecs: 218982458 +frame_id: "camera_color_optical_frame" +[INFO] [1733882989.652766]: received mask_obs with header seq: 538 +stamp: + secs: 1733882989 + nsecs: 653637647 +frame_id: '' and grasped_object_mask header seq: 39540 +stamp: + secs: 1733882989 + nsecs: 418820858 +frame_id: "camera_color_optical_frame" +[INFO] [1733882989.844128]: received mask_obs with header seq: 539 +stamp: + secs: 1733882989 + nsecs: 844943046 +frame_id: '' and grasped_object_mask header seq: 39546 +stamp: + secs: 1733882989 + nsecs: 618659258 +frame_id: "camera_color_optical_frame" +[INFO] [1733882990.034861]: received mask_obs with header seq: 540 +stamp: + secs: 1733882990 + nsecs: 35667181 +frame_id: '' and grasped_object_mask header seq: 39552 +stamp: + secs: 1733882989 + nsecs: 818498373 +frame_id: "camera_color_optical_frame" +[INFO] [1733882990.248011]: received mask_obs with header seq: 541 +stamp: + secs: 1733882990 + nsecs: 248704910 +frame_id: '' and grasped_object_mask header seq: 39558 +stamp: + secs: 1733882990 + nsecs: 18334866 +frame_id: "camera_color_optical_frame" +[INFO] [1733882990.451972]: received mask_obs with header seq: 542 +stamp: + secs: 1733882990 + nsecs: 452630996 +frame_id: '' and grasped_object_mask header seq: 39564 +stamp: + secs: 1733882990 + nsecs: 218169212 +frame_id: "camera_color_optical_frame" +[INFO] [1733882990.638939]: received mask_obs with header seq: 543 +stamp: + secs: 1733882990 + nsecs: 639698505 +frame_id: '' and grasped_object_mask header seq: 39570 +stamp: + secs: 1733882990 + nsecs: 418003321 +frame_id: "camera_color_optical_frame" +[INFO] [1733882990.842749]: received mask_obs with header seq: 544 +stamp: + secs: 1733882990 + nsecs: 843642950 +frame_id: '' and grasped_object_mask header seq: 39576 +stamp: + secs: 1733882990 + nsecs: 617837667 +frame_id: "camera_color_optical_frame" +[INFO] [1733882991.039415]: received mask_obs with header seq: 545 +stamp: + secs: 1733882991 + nsecs: 40328025 +frame_id: '' and grasped_object_mask header seq: 39582 +stamp: + secs: 1733882990 + nsecs: 817671537 +frame_id: "camera_color_optical_frame" +[INFO] [1733882991.240857]: received mask_obs with header seq: 546 +stamp: + secs: 1733882991 + nsecs: 241757631 +frame_id: '' and grasped_object_mask header seq: 39588 +stamp: + secs: 1733882991 + nsecs: 17507076 +frame_id: "camera_color_optical_frame" +[INFO] [1733882991.442936]: received mask_obs with header seq: 547 +stamp: + secs: 1733882991 + nsecs: 443448781 +frame_id: '' and grasped_object_mask header seq: 39594 +stamp: + secs: 1733882991 + nsecs: 217342138 +frame_id: "camera_color_optical_frame" +[INFO] [1733882991.651124]: received mask_obs with header seq: 548 +stamp: + secs: 1733882991 + nsecs: 652017831 +frame_id: '' and grasped_object_mask header seq: 39600 +stamp: + secs: 1733882991 + nsecs: 417176723 +frame_id: "camera_color_optical_frame" +[INFO] [1733882991.834174]: received mask_obs with header seq: 549 +stamp: + secs: 1733882991 + nsecs: 834767818 +frame_id: '' and grasped_object_mask header seq: 39606 +stamp: + secs: 1733882991 + nsecs: 617011547 +frame_id: "camera_color_optical_frame" +[INFO] [1733882992.046498]: received mask_obs with header seq: 550 +stamp: + secs: 1733882992 + nsecs: 47367334 +frame_id: '' and grasped_object_mask header seq: 39612 +stamp: + secs: 1733882991 + nsecs: 816845894 +frame_id: "camera_color_optical_frame" +[INFO] [1733882992.250119]: received mask_obs with header seq: 551 +stamp: + secs: 1733882992 + nsecs: 251020193 +frame_id: '' and grasped_object_mask header seq: 39618 +stamp: + secs: 1733882992 + nsecs: 16683340 +frame_id: "camera_color_optical_frame" +[INFO] [1733882992.456320]: received mask_obs with header seq: 552 +stamp: + secs: 1733882992 + nsecs: 457109451 +frame_id: '' and grasped_object_mask header seq: 39624 +stamp: + secs: 1733882992 + nsecs: 216524601 +frame_id: "camera_color_optical_frame" +[INFO] [1733882992.652031]: received mask_obs with header seq: 553 +stamp: + secs: 1733882992 + nsecs: 652771949 +frame_id: '' and grasped_object_mask header seq: 39630 +stamp: + secs: 1733882992 + nsecs: 416365147 +frame_id: "camera_color_optical_frame" +[INFO] [1733882992.847601]: received mask_obs with header seq: 554 +stamp: + secs: 1733882992 + nsecs: 848126173 +frame_id: '' and grasped_object_mask header seq: 39636 +stamp: + secs: 1733882992 + nsecs: 616205692 +frame_id: "camera_color_optical_frame" +[INFO] [1733882993.035531]: received mask_obs with header seq: 555 +stamp: + secs: 1733882993 + nsecs: 36165952 +frame_id: '' and grasped_object_mask header seq: 39642 +stamp: + secs: 1733882992 + nsecs: 816046715 +frame_id: "camera_color_optical_frame" +[INFO] [1733882993.235206]: received mask_obs with header seq: 556 +stamp: + secs: 1733882993 + nsecs: 236050844 +frame_id: '' and grasped_object_mask header seq: 39648 +stamp: + secs: 1733882993 + nsecs: 15881777 +frame_id: "camera_color_optical_frame" +[INFO] [1733882993.439841]: received mask_obs with header seq: 557 +stamp: + secs: 1733882993 + nsecs: 440672159 +frame_id: '' and grasped_object_mask header seq: 39654 +stamp: + secs: 1733882993 + nsecs: 215710878 +frame_id: "camera_color_optical_frame" +[INFO] [1733882993.636498]: received mask_obs with header seq: 558 +stamp: + secs: 1733882993 + nsecs: 637322187 +frame_id: '' and grasped_object_mask header seq: 39660 +stamp: + secs: 1733882993 + nsecs: 415540695 +frame_id: "camera_color_optical_frame" +[INFO] [1733882993.839381]: received mask_obs with header seq: 559 +stamp: + secs: 1733882993 + nsecs: 840332508 +frame_id: '' and grasped_object_mask header seq: 39666 +stamp: + secs: 1733882993 + nsecs: 615368843 +frame_id: "camera_color_optical_frame" +[INFO] [1733882994.044861]: received mask_obs with header seq: 560 +stamp: + secs: 1733882994 + nsecs: 45722484 +frame_id: '' and grasped_object_mask header seq: 39672 +stamp: + secs: 1733882993 + nsecs: 815196514 +frame_id: "camera_color_optical_frame" +[INFO] [1733882994.238032]: received mask_obs with header seq: 561 +stamp: + secs: 1733882994 + nsecs: 238701343 +frame_id: '' and grasped_object_mask header seq: 39678 +stamp: + secs: 1733882994 + nsecs: 15028238 +frame_id: "camera_color_optical_frame" +[INFO] [1733882994.446261]: received mask_obs with header seq: 562 +stamp: + secs: 1733882994 + nsecs: 447117567 +frame_id: '' and grasped_object_mask header seq: 39684 +stamp: + secs: 1733882994 + nsecs: 214864969 +frame_id: "camera_color_optical_frame" +[INFO] [1733882994.640620]: received mask_obs with header seq: 563 +stamp: + secs: 1733882994 + nsecs: 640420675 +frame_id: '' and grasped_object_mask header seq: 39690 +stamp: + secs: 1733882994 + nsecs: 414701462 +frame_id: "camera_color_optical_frame" +[INFO] [1733882994.840210]: received mask_obs with header seq: 564 +stamp: + secs: 1733882994 + nsecs: 840333223 +frame_id: '' and grasped_object_mask header seq: 39696 +stamp: + secs: 1733882994 + nsecs: 614538908 +frame_id: "camera_color_optical_frame" +[INFO] [1733882995.037001]: received mask_obs with header seq: 565 +stamp: + secs: 1733882995 + nsecs: 37460565 +frame_id: '' and grasped_object_mask header seq: 39702 +stamp: + secs: 1733882994 + nsecs: 814375162 +frame_id: "camera_color_optical_frame" +[INFO] [1733882995.235118]: received mask_obs with header seq: 566 +stamp: + secs: 1733882995 + nsecs: 236046552 +frame_id: '' and grasped_object_mask header seq: 39708 +stamp: + secs: 1733882995 + nsecs: 14209509 +frame_id: "camera_color_optical_frame" +[INFO] [1733882995.454906]: received mask_obs with header seq: 567 +stamp: + secs: 1733882995 + nsecs: 455580234 +frame_id: '' and grasped_object_mask header seq: 39714 +stamp: + secs: 1733882995 + nsecs: 214041471 +frame_id: "camera_color_optical_frame" +[INFO] [1733882995.675060]: received mask_obs with header seq: 568 +stamp: + secs: 1733882995 + nsecs: 674569368 +frame_id: '' and grasped_object_mask header seq: 39720 +stamp: + secs: 1733882995 + nsecs: 413873434 +frame_id: "camera_color_optical_frame" +[INFO] [1733882995.869910]: received mask_obs with header seq: 569 +stamp: + secs: 1733882995 + nsecs: 870277643 +frame_id: '' and grasped_object_mask header seq: 39727 +stamp: + secs: 1733882995 + nsecs: 647010803 +frame_id: "camera_color_optical_frame" +[INFO] [1733882996.048927]: received mask_obs with header seq: 570 +stamp: + secs: 1733882996 + nsecs: 49722194 +frame_id: '' and grasped_object_mask header seq: 39732 +stamp: + secs: 1733882995 + nsecs: 813536167 +frame_id: "camera_color_optical_frame" +[INFO] [1733882996.244311]: received mask_obs with header seq: 571 +stamp: + secs: 1733882996 + nsecs: 245156288 +frame_id: '' and grasped_object_mask header seq: 39738 +stamp: + secs: 1733882996 + nsecs: 13374805 +frame_id: "camera_color_optical_frame" +[INFO] [1733882996.447204]: received mask_obs with header seq: 572 +stamp: + secs: 1733882996 + nsecs: 447950601 +frame_id: '' and grasped_object_mask header seq: 39744 +stamp: + secs: 1733882996 + nsecs: 213218689 +frame_id: "camera_color_optical_frame" +[INFO] [1733882996.640351]: received mask_obs with header seq: 573 +stamp: + secs: 1733882996 + nsecs: 641144752 +frame_id: '' and grasped_object_mask header seq: 39750 +stamp: + secs: 1733882996 + nsecs: 413063049 +frame_id: "camera_color_optical_frame" +[INFO] [1733882996.840109]: received mask_obs with header seq: 574 +stamp: + secs: 1733882996 + nsecs: 840967178 +frame_id: '' and grasped_object_mask header seq: 39756 +stamp: + secs: 1733882996 + nsecs: 612907648 +frame_id: "camera_color_optical_frame" +[INFO] [1733882997.040877]: received mask_obs with header seq: 575 +stamp: + secs: 1733882997 + nsecs: 41819095 +frame_id: '' and grasped_object_mask header seq: 39762 +stamp: + secs: 1733882996 + nsecs: 812752485 +frame_id: "camera_color_optical_frame" +[INFO] [1733882997.251767]: received mask_obs with header seq: 576 +stamp: + secs: 1733882997 + nsecs: 252618789 +frame_id: '' and grasped_object_mask header seq: 39768 +stamp: + secs: 1733882997 + nsecs: 12595654 +frame_id: "camera_color_optical_frame" +[INFO] [1733882997.444350]: received mask_obs with header seq: 577 +stamp: + secs: 1733882997 + nsecs: 445230245 +frame_id: '' and grasped_object_mask header seq: 39774 +stamp: + secs: 1733882997 + nsecs: 212438345 +frame_id: "camera_color_optical_frame" +[INFO] [1733882997.641463]: received mask_obs with header seq: 578 +stamp: + secs: 1733882997 + nsecs: 642207384 +frame_id: '' and grasped_object_mask header seq: 39780 +stamp: + secs: 1733882997 + nsecs: 412279844 +frame_id: "camera_color_optical_frame" +[INFO] [1733882997.837665]: received mask_obs with header seq: 579 +stamp: + secs: 1733882997 + nsecs: 838302612 +frame_id: '' and grasped_object_mask header seq: 39786 +stamp: + secs: 1733882997 + nsecs: 612121582 +frame_id: "camera_color_optical_frame" +[INFO] [1733882998.037341]: received mask_obs with header seq: 580 +stamp: + secs: 1733882998 + nsecs: 35892963 +frame_id: '' and grasped_object_mask header seq: 39792 +stamp: + secs: 1733882997 + nsecs: 811963558 +frame_id: "camera_color_optical_frame" +[INFO] [1733882998.236020]: received mask_obs with header seq: 581 +stamp: + secs: 1733882998 + nsecs: 236891984 +frame_id: '' and grasped_object_mask header seq: 39798 +stamp: + secs: 1733882998 + nsecs: 11806726 +frame_id: "camera_color_optical_frame" +[INFO] [1733882998.443982]: received mask_obs with header seq: 582 +stamp: + secs: 1733882998 + nsecs: 444771766 +frame_id: '' and grasped_object_mask header seq: 39804 +stamp: + secs: 1733882998 + nsecs: 211650610 +frame_id: "camera_color_optical_frame" +[INFO] [1733882998.640382]: received mask_obs with header seq: 583 +stamp: + secs: 1733882998 + nsecs: 641317605 +frame_id: '' and grasped_object_mask header seq: 39810 +stamp: + secs: 1733882998 + nsecs: 411496162 +frame_id: "camera_color_optical_frame" +[INFO] [1733882998.835207]: received mask_obs with header seq: 584 +stamp: + secs: 1733882998 + nsecs: 835690736 +frame_id: '' and grasped_object_mask header seq: 39816 +stamp: + secs: 1733882998 + nsecs: 611340523 +frame_id: "camera_color_optical_frame" +[INFO] [1733882999.037221]: received mask_obs with header seq: 585 +stamp: + secs: 1733882999 + nsecs: 38119077 +frame_id: '' and grasped_object_mask header seq: 39822 +stamp: + secs: 1733882998 + nsecs: 811185598 +frame_id: "camera_color_optical_frame" +[INFO] [1733882999.232694]: received mask_obs with header seq: 586 +stamp: + secs: 1733882999 + nsecs: 233396053 +frame_id: '' and grasped_object_mask header seq: 39828 +stamp: + secs: 1733882999 + nsecs: 11032581 +frame_id: "camera_color_optical_frame" +[INFO] [1733882999.438339]: received mask_obs with header seq: 587 +stamp: + secs: 1733882999 + nsecs: 439226865 +frame_id: '' and grasped_object_mask header seq: 39834 +stamp: + secs: 1733882999 + nsecs: 210881948 +frame_id: "camera_color_optical_frame" +[INFO] [1733882999.639158]: received mask_obs with header seq: 588 +stamp: + secs: 1733882999 + nsecs: 639953374 +frame_id: '' and grasped_object_mask header seq: 39840 +stamp: + secs: 1733882999 + nsecs: 410731554 +frame_id: "camera_color_optical_frame" +[INFO] [1733882999.841585]: received mask_obs with header seq: 589 +stamp: + secs: 1733882999 + nsecs: 842472791 +frame_id: '' and grasped_object_mask header seq: 39846 +stamp: + secs: 1733882999 + nsecs: 610582829 +frame_id: "camera_color_optical_frame" +[INFO] [1733883000.034769]: received mask_obs with header seq: 590 +stamp: + secs: 1733883000 + nsecs: 35015344 +frame_id: '' and grasped_object_mask header seq: 39852 +stamp: + secs: 1733882999 + nsecs: 810433388 +frame_id: "camera_color_optical_frame" +[INFO] [1733883000.236456]: received mask_obs with header seq: 591 +stamp: + secs: 1733883000 + nsecs: 237433433 +frame_id: '' and grasped_object_mask header seq: 39858 +stamp: + secs: 1733883000 + nsecs: 10277510 +frame_id: "camera_color_optical_frame" +[INFO] [1733883000.437034]: received mask_obs with header seq: 592 +stamp: + secs: 1733883000 + nsecs: 437916994 +frame_id: '' and grasped_object_mask header seq: 39864 +stamp: + secs: 1733883000 + nsecs: 210114241 +frame_id: "camera_color_optical_frame" +[INFO] [1733883000.637746]: received mask_obs with header seq: 593 +stamp: + secs: 1733883000 + nsecs: 638127326 +frame_id: '' and grasped_object_mask header seq: 39870 +stamp: + secs: 1733883000 + nsecs: 409950972 +frame_id: "camera_color_optical_frame" +[INFO] [1733883000.836298]: received mask_obs with header seq: 594 +stamp: + secs: 1733883000 + nsecs: 837185621 +frame_id: '' and grasped_object_mask header seq: 39876 +stamp: + secs: 1733883000 + nsecs: 609787464 +frame_id: "camera_color_optical_frame" +[INFO] [1733883001.036034]: received mask_obs with header seq: 595 +stamp: + secs: 1733883001 + nsecs: 36351919 +frame_id: '' and grasped_object_mask header seq: 39882 +stamp: + secs: 1733883000 + nsecs: 809624910 +frame_id: "camera_color_optical_frame" +[INFO] [1733883001.245102]: received mask_obs with header seq: 596 +stamp: + secs: 1733883001 + nsecs: 245862483 +frame_id: '' and grasped_object_mask header seq: 39888 +stamp: + secs: 1733883001 + nsecs: 9462357 +frame_id: "camera_color_optical_frame" +[INFO] [1733883001.438123]: received mask_obs with header seq: 597 +stamp: + secs: 1733883001 + nsecs: 438766241 +frame_id: '' and grasped_object_mask header seq: 39894 +stamp: + secs: 1733883001 + nsecs: 209301233 +frame_id: "camera_color_optical_frame" +[INFO] [1733883001.636743]: received mask_obs with header seq: 598 +stamp: + secs: 1733883001 + nsecs: 637579202 +frame_id: '' and grasped_object_mask header seq: 39900 +stamp: + secs: 1733883001 + nsecs: 409139872 +frame_id: "camera_color_optical_frame" +[INFO] [1733883001.835870]: received mask_obs with header seq: 599 +stamp: + secs: 1733883001 + nsecs: 836532831 +frame_id: '' and grasped_object_mask header seq: 39906 +stamp: + secs: 1733883001 + nsecs: 608978748 +frame_id: "camera_color_optical_frame" +[INFO] [1733883002.041073]: received mask_obs with header seq: 600 +stamp: + secs: 1733883002 + nsecs: 41754484 +frame_id: '' and grasped_object_mask header seq: 39912 +stamp: + secs: 1733883001 + nsecs: 808817625 +frame_id: "camera_color_optical_frame" +[INFO] [1733883002.237072]: received mask_obs with header seq: 601 +stamp: + secs: 1733883002 + nsecs: 237576484 +frame_id: '' and grasped_object_mask header seq: 39918 +stamp: + secs: 1733883002 + nsecs: 8671284 +frame_id: "camera_color_optical_frame" +[INFO] [1733883002.444077]: received mask_obs with header seq: 602 +stamp: + secs: 1733883002 + nsecs: 444773435 +frame_id: '' and grasped_object_mask header seq: 39924 +stamp: + secs: 1733883002 + nsecs: 208540678 +frame_id: "camera_color_optical_frame" +[INFO] [1733883002.645746]: received mask_obs with header seq: 603 +stamp: + secs: 1733883002 + nsecs: 646395921 +frame_id: '' and grasped_object_mask header seq: 39930 +stamp: + secs: 1733883002 + nsecs: 408411026 +frame_id: "camera_color_optical_frame" +[INFO] [1733883002.846055]: received mask_obs with header seq: 604 +stamp: + secs: 1733883002 + nsecs: 846941471 +frame_id: '' and grasped_object_mask header seq: 39936 +stamp: + secs: 1733883002 + nsecs: 608282566 +frame_id: "camera_color_optical_frame" +[INFO] [1733883003.049752]: received mask_obs with header seq: 605 +stamp: + secs: 1733883003 + nsecs: 50101518 +frame_id: '' and grasped_object_mask header seq: 39942 +stamp: + secs: 1733883002 + nsecs: 808155060 +frame_id: "camera_color_optical_frame" +[INFO] [1733883003.240892]: received mask_obs with header seq: 606 +stamp: + secs: 1733883003 + nsecs: 241748809 +frame_id: '' and grasped_object_mask header seq: 39948 +stamp: + secs: 1733883003 + nsecs: 8013248 +frame_id: "camera_color_optical_frame" +[INFO] [1733883003.440418]: received mask_obs with header seq: 607 +stamp: + secs: 1733883003 + nsecs: 440999507 +frame_id: '' and grasped_object_mask header seq: 39954 +stamp: + secs: 1733883003 + nsecs: 207854033 +frame_id: "camera_color_optical_frame" +[INFO] [1733883003.646784]: received mask_obs with header seq: 608 +stamp: + secs: 1733883003 + nsecs: 647576570 +frame_id: '' and grasped_object_mask header seq: 39960 +stamp: + secs: 1733883003 + nsecs: 407696009 +frame_id: "camera_color_optical_frame" +[INFO] [1733883003.837464]: received mask_obs with header seq: 609 +stamp: + secs: 1733883003 + nsecs: 838406562 +frame_id: '' and grasped_object_mask header seq: 39966 +stamp: + secs: 1733883003 + nsecs: 607537270 +frame_id: "camera_color_optical_frame" +[INFO] [1733883004.040832]: received mask_obs with header seq: 610 +stamp: + secs: 1733883004 + nsecs: 41711568 +frame_id: '' and grasped_object_mask header seq: 39972 +stamp: + secs: 1733883003 + nsecs: 807379007 +frame_id: "camera_color_optical_frame" +[INFO] [1733883004.234583]: received mask_obs with header seq: 611 +stamp: + secs: 1733883004 + nsecs: 235076904 +frame_id: '' and grasped_object_mask header seq: 39978 +stamp: + secs: 1733883004 + nsecs: 7218122 +frame_id: "camera_color_optical_frame" +[INFO] [1733883004.441224]: received mask_obs with header seq: 612 +stamp: + secs: 1733883004 + nsecs: 442162275 +frame_id: '' and grasped_object_mask header seq: 39984 +stamp: + secs: 1733883004 + nsecs: 207054615 +frame_id: "camera_color_optical_frame" +[INFO] [1733883004.641501]: received mask_obs with header seq: 613 +stamp: + secs: 1733883004 + nsecs: 642345190 +frame_id: '' and grasped_object_mask header seq: 39990 +stamp: + secs: 1733883004 + nsecs: 406891108 +frame_id: "camera_color_optical_frame" +[INFO] [1733883004.852892]: received mask_obs with header seq: 614 +stamp: + secs: 1733883004 + nsecs: 853773832 +frame_id: '' and grasped_object_mask header seq: 39996 +stamp: + secs: 1733883004 + nsecs: 606728315 +frame_id: "camera_color_optical_frame" +[INFO] [1733883005.065852]: received mask_obs with header seq: 615 +stamp: + secs: 1733883005 + nsecs: 66716194 +frame_id: '' and grasped_object_mask header seq: 40002 +stamp: + secs: 1733883004 + nsecs: 806564808 +frame_id: "camera_color_optical_frame" +[INFO] [1733883005.273036]: received mask_obs with header seq: 616 +stamp: + secs: 1733883005 + nsecs: 273846626 +frame_id: '' and grasped_object_mask header seq: 40008 +stamp: + secs: 1733883005 + nsecs: 6399393 +frame_id: "camera_color_optical_frame" +[INFO] [1733883005.478649]: received mask_obs with header seq: 617 +stamp: + secs: 1733883005 + nsecs: 479404687 +frame_id: '' and grasped_object_mask header seq: 40015 +stamp: + secs: 1733883005 + nsecs: 239538431 +frame_id: "camera_color_optical_frame" +[INFO] [1733883005.671099]: received mask_obs with header seq: 618 +stamp: + secs: 1733883005 + nsecs: 671912431 +frame_id: '' and grasped_object_mask header seq: 40021 +stamp: + secs: 1733883005 + nsecs: 439371109 +frame_id: "camera_color_optical_frame" +[INFO] [1733883005.884880]: received mask_obs with header seq: 619 +stamp: + secs: 1733883005 + nsecs: 885801315 +frame_id: '' and grasped_object_mask header seq: 40027 +stamp: + secs: 1733883005 + nsecs: 639203548 +frame_id: "camera_color_optical_frame" +[INFO] [1733883006.076405]: received mask_obs with header seq: 620 +stamp: + secs: 1733883006 + nsecs: 77027320 +frame_id: '' and grasped_object_mask header seq: 40033 +stamp: + secs: 1733883005 + nsecs: 839035749 +frame_id: "camera_color_optical_frame" +[INFO] [1733883006.284471]: received mask_obs with header seq: 621 +stamp: + secs: 1733883006 + nsecs: 285383462 +frame_id: '' and grasped_object_mask header seq: 40039 +stamp: + secs: 1733883006 + nsecs: 38868189 +frame_id: "camera_color_optical_frame" +[INFO] [1733883006.478817]: received mask_obs with header seq: 622 +stamp: + secs: 1733883006 + nsecs: 479475498 +frame_id: '' and grasped_object_mask header seq: 40045 +stamp: + secs: 1733883006 + nsecs: 238700390 +frame_id: "camera_color_optical_frame" +[INFO] [1733883006.678516]: received mask_obs with header seq: 623 +stamp: + secs: 1733883006 + nsecs: 679447889 +frame_id: '' and grasped_object_mask header seq: 40051 +stamp: + secs: 1733883006 + nsecs: 438533306 +frame_id: "camera_color_optical_frame" +[INFO] [1733883006.885924]: received mask_obs with header seq: 624 +stamp: + secs: 1733883006 + nsecs: 886447429 +frame_id: '' and grasped_object_mask header seq: 40057 +stamp: + secs: 1733883006 + nsecs: 638364792 +frame_id: "camera_color_optical_frame" +[INFO] [1733883007.077788]: received mask_obs with header seq: 625 +stamp: + secs: 1733883007 + nsecs: 78685760 +frame_id: '' and grasped_object_mask header seq: 40063 +stamp: + secs: 1733883006 + nsecs: 838195801 +frame_id: "camera_color_optical_frame" +[INFO] [1733883007.279273]: received mask_obs with header seq: 626 +stamp: + secs: 1733883007 + nsecs: 280079364 +frame_id: '' and grasped_object_mask header seq: 40069 +stamp: + secs: 1733883007 + nsecs: 38028717 +frame_id: "camera_color_optical_frame" +[INFO] [1733883007.483844]: received mask_obs with header seq: 627 +stamp: + secs: 1733883007 + nsecs: 484565734 +frame_id: '' and grasped_object_mask header seq: 40075 +stamp: + secs: 1733883007 + nsecs: 237862825 +frame_id: "camera_color_optical_frame" +[INFO] [1733883007.687912]: received mask_obs with header seq: 628 +stamp: + secs: 1733883007 + nsecs: 688732624 +frame_id: '' and grasped_object_mask header seq: 40081 +stamp: + secs: 1733883007 + nsecs: 437696457 +frame_id: "camera_color_optical_frame" +[INFO] [1733883007.874056]: received mask_obs with header seq: 629 +stamp: + secs: 1733883007 + nsecs: 874939203 +frame_id: '' and grasped_object_mask header seq: 40087 +stamp: + secs: 1733883007 + nsecs: 637531281 +frame_id: "camera_color_optical_frame" +[INFO] [1733883008.073511]: received mask_obs with header seq: 630 +stamp: + secs: 1733883008 + nsecs: 74264049 +frame_id: '' and grasped_object_mask header seq: 40093 +stamp: + secs: 1733883007 + nsecs: 837364435 +frame_id: "camera_color_optical_frame" +[INFO] [1733883008.271866]: received mask_obs with header seq: 631 +stamp: + secs: 1733883008 + nsecs: 272708654 +frame_id: '' and grasped_object_mask header seq: 40099 +stamp: + secs: 1733883008 + nsecs: 37199259 +frame_id: "camera_color_optical_frame" +[INFO] [1733883008.473479]: received mask_obs with header seq: 632 +stamp: + secs: 1733883008 + nsecs: 474390029 +frame_id: '' and grasped_object_mask header seq: 40105 +stamp: + secs: 1733883008 + nsecs: 237034798 +frame_id: "camera_color_optical_frame" +[INFO] [1733883008.670142]: received mask_obs with header seq: 633 +stamp: + secs: 1733883008 + nsecs: 670746564 +frame_id: '' and grasped_object_mask header seq: 40111 +stamp: + secs: 1733883008 + nsecs: 436870337 +frame_id: "camera_color_optical_frame" +[INFO] [1733883008.872947]: received mask_obs with header seq: 634 +stamp: + secs: 1733883008 + nsecs: 873882532 +frame_id: '' and grasped_object_mask header seq: 40117 +stamp: + secs: 1733883008 + nsecs: 636705875 +frame_id: "camera_color_optical_frame" +[INFO] [1733883009.069458]: received mask_obs with header seq: 635 +stamp: + secs: 1733883009 + nsecs: 69400072 +frame_id: '' and grasped_object_mask header seq: 40123 +stamp: + secs: 1733883008 + nsecs: 836541891 +frame_id: "camera_color_optical_frame" +[INFO] [1733883009.265201]: received mask_obs with header seq: 636 +stamp: + secs: 1733883009 + nsecs: 266047239 +frame_id: '' and grasped_object_mask header seq: 40129 +stamp: + secs: 1733883009 + nsecs: 36375046 +frame_id: "camera_color_optical_frame" +[INFO] [1733883009.447315]: received mask_obs with header seq: 637 +stamp: + secs: 1733883009 + nsecs: 448097229 +frame_id: '' and grasped_object_mask header seq: 40134 +stamp: + secs: 1733883009 + nsecs: 202900887 +frame_id: "camera_color_optical_frame" +[INFO] [1733883009.637640]: received mask_obs with header seq: 638 +stamp: + secs: 1733883009 + nsecs: 638221263 +frame_id: '' and grasped_object_mask header seq: 40140 +stamp: + secs: 1733883009 + nsecs: 402733088 +frame_id: "camera_color_optical_frame" +[INFO] [1733883009.840062]: received mask_obs with header seq: 639 +stamp: + secs: 1733883009 + nsecs: 840966939 +frame_id: '' and grasped_object_mask header seq: 40146 +stamp: + secs: 1733883009 + nsecs: 602563620 +frame_id: "camera_color_optical_frame" +[INFO] [1733883010.039051]: received mask_obs with header seq: 640 +stamp: + secs: 1733883010 + nsecs: 39928436 +frame_id: '' and grasped_object_mask header seq: 40152 +stamp: + secs: 1733883009 + nsecs: 802394390 +frame_id: "camera_color_optical_frame" +[INFO] [1733883010.237635]: received mask_obs with header seq: 641 +stamp: + secs: 1733883010 + nsecs: 238132953 +frame_id: '' and grasped_object_mask header seq: 40158 +stamp: + secs: 1733883010 + nsecs: 2225637 +frame_id: "camera_color_optical_frame" +[INFO] [1733883010.438774]: received mask_obs with header seq: 642 +stamp: + secs: 1733883010 + nsecs: 439603567 +frame_id: '' and grasped_object_mask header seq: 40164 +stamp: + secs: 1733883010 + nsecs: 202058315 +frame_id: "camera_color_optical_frame" +[INFO] [1733883010.642000]: received mask_obs with header seq: 643 +stamp: + secs: 1733883010 + nsecs: 642556905 +frame_id: '' and grasped_object_mask header seq: 40170 +stamp: + secs: 1733883010 + nsecs: 401890755 +frame_id: "camera_color_optical_frame" +[INFO] [1733883010.843387]: received mask_obs with header seq: 644 +stamp: + secs: 1733883010 + nsecs: 844290018 +frame_id: '' and grasped_object_mask header seq: 40176 +stamp: + secs: 1733883010 + nsecs: 601722956 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is 100738503 for nearest: 1733883010902294137 - target: 1733883010801555634 at idx 249 +[INFO] [1733883011.041939]: received mask_obs with header seq: 645 +stamp: + secs: 1733883011 + nsecs: 42489767 +frame_id: '' and grasped_object_mask header seq: 40182 +stamp: + secs: 1733883010 + nsecs: 801555634 +frame_id: "camera_color_optical_frame" +[INFO] [1733883011.238333]: received mask_obs with header seq: 646 +stamp: + secs: 1733883011 + nsecs: 239225864 +frame_id: '' and grasped_object_mask header seq: 40188 +stamp: + secs: 1733883011 + nsecs: 1388788 +frame_id: "camera_color_optical_frame" +[INFO] [1733883011.448368]: received mask_obs with header seq: 647 +stamp: + secs: 1733883011 + nsecs: 449202775 +frame_id: '' and grasped_object_mask header seq: 40194 +stamp: + secs: 1733883011 + nsecs: 201223135 +frame_id: "camera_color_optical_frame" +[INFO] [1733883011.644617]: received mask_obs with header seq: 648 +stamp: + secs: 1733883011 + nsecs: 645199298 +frame_id: '' and grasped_object_mask header seq: 40200 +stamp: + secs: 1733883011 + nsecs: 401057243 +frame_id: "camera_color_optical_frame" +[INFO] [1733883011.836370]: received mask_obs with header seq: 649 +stamp: + secs: 1733883011 + nsecs: 837244510 +frame_id: '' and grasped_object_mask header seq: 40206 +stamp: + secs: 1733883011 + nsecs: 600890875 +frame_id: "camera_color_optical_frame" +[INFO] [1733883012.060653]: received mask_obs with header seq: 650 +stamp: + secs: 1733883012 + nsecs: 61558723 +frame_id: '' and grasped_object_mask header seq: 40212 +stamp: + secs: 1733883011 + nsecs: 800724268 +frame_id: "camera_color_optical_frame" +[INFO] [1733883012.242988]: received mask_obs with header seq: 651 +stamp: + secs: 1733883012 + nsecs: 243913650 +frame_id: '' and grasped_object_mask header seq: 40218 +stamp: + secs: 1733883012 + nsecs: 561476 +frame_id: "camera_color_optical_frame" +[INFO] [1733883012.445169]: received mask_obs with header seq: 652 +stamp: + secs: 1733883012 + nsecs: 446041822 +frame_id: '' and grasped_object_mask header seq: 40224 +stamp: + secs: 1733883012 + nsecs: 200402975 +frame_id: "camera_color_optical_frame" +[INFO] [1733883012.643172]: received mask_obs with header seq: 653 +stamp: + secs: 1733883012 + nsecs: 644080400 +frame_id: '' and grasped_object_mask header seq: 40230 +stamp: + secs: 1733883012 + nsecs: 400244474 +frame_id: "camera_color_optical_frame" +[INFO] [1733883012.841066]: received mask_obs with header seq: 654 +stamp: + secs: 1733883012 + nsecs: 841974973 +frame_id: '' and grasped_object_mask header seq: 40236 +stamp: + secs: 1733883012 + nsecs: 600086212 +frame_id: "camera_color_optical_frame" +[INFO] [1733883012.880501]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1733883013.041724]: received mask_obs with header seq: 655 +stamp: + secs: 1733883013 + nsecs: 39988756 +frame_id: '' and grasped_object_mask header seq: 40242 +stamp: + secs: 1733883012 + nsecs: 799927711 +frame_id: "camera_color_optical_frame" +[INFO] [1733883013.614776]: Stored episode 3. +[INFO] [1733883013.619309]: Saving video... +[INFO] [1733883013.984778]: Video saved! +Starting episode 3 +[INFO] [1733883014.028715]: resetting environment +[INFO] [1733883014.042045]: cleared current plan +[INFO] [1733883014.048223]: moving to home +[INFO] [1733883016.857514]: reached home +[INFO] [1733883016.862701]: reset action history +[INFO] [1733883019.647636]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1733883022.533171]: resetting environment +[INFO] [1733883022.549977]: cleared current plan +[INFO] [1733883022.554803]: moving to home +[INFO] [1733883023.571791]: reached home +[INFO] [1733883023.574949]: reset action history +[INFO] [1733883026.294622]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/episode_rosbags/episode_3_2024-12-10-21-10-26.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-600897.log +started roslaunch server http://158.130.50.37:35391/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [601606] +process[print_text-11]: started with pid [601607] +process[rosbag_record-12]: started with pid [601631] +started bagging! +[INFO] [1733883026.987097]: received mask_obs with header seq: 656 +stamp: + secs: 1733883026 + nsecs: 987187623 +frame_id: '' and grasped_object_mask header seq: 40662 +stamp: + secs: 1733883026 + nsecs: 796250343 +frame_id: "camera_color_optical_frame" +[INFO] [1733883027.187410]: received mask_obs with header seq: 657 +stamp: + secs: 1733883027 + nsecs: 187712430 +frame_id: '' and grasped_object_mask header seq: 40668 +stamp: + secs: 1733883026 + nsecs: 995815992 +frame_id: "camera_color_optical_frame" +[INFO] [1733883027.387372]: received mask_obs with header seq: 658 +stamp: + secs: 1733883027 + nsecs: 386941432 +frame_id: '' and grasped_object_mask header seq: 40674 +stamp: + secs: 1733883027 + nsecs: 195435762 +frame_id: "camera_color_optical_frame" +[INFO] [1733883027.601449]: received mask_obs with header seq: 659 +stamp: + secs: 1733883027 + nsecs: 602356433 +frame_id: '' and grasped_object_mask header seq: 40680 +stamp: + secs: 1733883027 + nsecs: 395044327 +frame_id: "camera_color_optical_frame" +[INFO] [1733883027.801857]: received mask_obs with header seq: 660 +stamp: + secs: 1733883027 + nsecs: 802163600 +frame_id: '' and grasped_object_mask header seq: 40686 +stamp: + secs: 1733883027 + nsecs: 594641685 +frame_id: "camera_color_optical_frame" +[INFO] [1733883027.999908]: received mask_obs with header seq: 661 +stamp: + secs: 1733883028 + nsecs: 673294 +frame_id: '' and grasped_object_mask header seq: 40692 +stamp: + secs: 1733883027 + nsecs: 794227600 +frame_id: "camera_color_optical_frame" +[INFO] [1733883028.203870]: received mask_obs with header seq: 662 +stamp: + secs: 1733883028 + nsecs: 203710556 +frame_id: '' and grasped_object_mask header seq: 40698 +stamp: + secs: 1733883027 + nsecs: 993803263 +frame_id: "camera_color_optical_frame" +[INFO] [1733883028.417335]: received mask_obs with header seq: 663 +stamp: + secs: 1733883028 + nsecs: 418273687 +frame_id: '' and grasped_object_mask header seq: 40704 +stamp: + secs: 1733883028 + nsecs: 193422556 +frame_id: "camera_color_optical_frame" +[INFO] [1733883028.613046]: received mask_obs with header seq: 664 +stamp: + secs: 1733883028 + nsecs: 613351583 +frame_id: '' and grasped_object_mask header seq: 40710 +stamp: + secs: 1733883028 + nsecs: 393030405 +frame_id: "camera_color_optical_frame" +[INFO] [1733883028.819163]: received mask_obs with header seq: 665 +stamp: + secs: 1733883028 + nsecs: 820078372 +frame_id: '' and grasped_object_mask header seq: 40716 +stamp: + secs: 1733883028 + nsecs: 592627525 +frame_id: "camera_color_optical_frame" +[INFO] [1733883029.024932]: received mask_obs with header seq: 666 +stamp: + secs: 1733883029 + nsecs: 25716543 +frame_id: '' and grasped_object_mask header seq: 40722 +stamp: + secs: 1733883028 + nsecs: 792212963 +frame_id: "camera_color_optical_frame" +[INFO] [1733883029.216795]: received mask_obs with header seq: 667 +stamp: + secs: 1733883029 + nsecs: 217761516 +frame_id: '' and grasped_object_mask header seq: 40728 +stamp: + secs: 1733883028 + nsecs: 991787195 +frame_id: "camera_color_optical_frame" +[INFO] [1733883029.427017]: received mask_obs with header seq: 668 +stamp: + secs: 1733883029 + nsecs: 427079439 +frame_id: '' and grasped_object_mask header seq: 40734 +stamp: + secs: 1733883029 + nsecs: 191401958 +frame_id: "camera_color_optical_frame" +[INFO] [1733883029.618660]: received mask_obs with header seq: 669 +stamp: + secs: 1733883029 + nsecs: 619529008 +frame_id: '' and grasped_object_mask header seq: 40740 +stamp: + secs: 1733883029 + nsecs: 391004086 +frame_id: "camera_color_optical_frame" +[INFO] [1733883029.817535]: received mask_obs with header seq: 670 +stamp: + secs: 1733883029 + nsecs: 818487405 +frame_id: '' and grasped_object_mask header seq: 40746 +stamp: + secs: 1733883029 + nsecs: 590594769 +frame_id: "camera_color_optical_frame" +[INFO] [1733883030.023374]: received mask_obs with header seq: 671 +stamp: + secs: 1733883030 + nsecs: 23981332 +frame_id: '' and grasped_object_mask header seq: 40752 +stamp: + secs: 1733883029 + nsecs: 790173769 +frame_id: "camera_color_optical_frame" +[INFO] [1733883030.223175]: received mask_obs with header seq: 672 +stamp: + secs: 1733883030 + nsecs: 223808288 +frame_id: '' and grasped_object_mask header seq: 40758 +stamp: + secs: 1733883029 + nsecs: 989740849 +frame_id: "camera_color_optical_frame" +[INFO] [1733883030.421453]: received mask_obs with header seq: 673 +stamp: + secs: 1733883030 + nsecs: 422243595 +frame_id: '' and grasped_object_mask header seq: 40764 +stamp: + secs: 1733883030 + nsecs: 189353704 +frame_id: "camera_color_optical_frame" +[INFO] [1733883030.627925]: received mask_obs with header seq: 674 +stamp: + secs: 1733883030 + nsecs: 628900289 +frame_id: '' and grasped_object_mask header seq: 40770 +stamp: + secs: 1733883030 + nsecs: 388955832 +frame_id: "camera_color_optical_frame" +[INFO] [1733883030.827872]: received mask_obs with header seq: 675 +stamp: + secs: 1733883030 + nsecs: 828637599 +frame_id: '' and grasped_object_mask header seq: 40776 +stamp: + secs: 1733883030 + nsecs: 588544607 +frame_id: "camera_color_optical_frame" +[INFO] [1733883031.020618]: received mask_obs with header seq: 676 +stamp: + secs: 1733883031 + nsecs: 21423578 +frame_id: '' and grasped_object_mask header seq: 40782 +stamp: + secs: 1733883030 + nsecs: 788122177 +frame_id: "camera_color_optical_frame" +[INFO] [1733883031.218648]: received mask_obs with header seq: 677 +stamp: + secs: 1733883031 + nsecs: 219496011 +frame_id: '' and grasped_object_mask header seq: 40788 +stamp: + secs: 1733883030 + nsecs: 987687588 +frame_id: "camera_color_optical_frame" +[INFO] [1733883031.417593]: received mask_obs with header seq: 678 +stamp: + secs: 1733883031 + nsecs: 418376445 +frame_id: '' and grasped_object_mask header seq: 40794 +stamp: + secs: 1733883031 + nsecs: 187302351 +frame_id: "camera_color_optical_frame" +[INFO] [1733883031.613167]: received mask_obs with header seq: 679 +stamp: + secs: 1733883031 + nsecs: 612986326 +frame_id: '' and grasped_object_mask header seq: 40800 +stamp: + secs: 1733883031 + nsecs: 386905193 +frame_id: "camera_color_optical_frame" +[INFO] [1733883031.810258]: received mask_obs with header seq: 680 +stamp: + secs: 1733883031 + nsecs: 810969114 +frame_id: '' and grasped_object_mask header seq: 40806 +stamp: + secs: 1733883031 + nsecs: 586496830 +frame_id: "camera_color_optical_frame" +[INFO] [1733883032.010499]: received mask_obs with header seq: 681 +stamp: + secs: 1733883032 + nsecs: 11319398 +frame_id: '' and grasped_object_mask header seq: 40812 +stamp: + secs: 1733883031 + nsecs: 786077499 +frame_id: "camera_color_optical_frame" +[INFO] [1733883032.205304]: received mask_obs with header seq: 682 +stamp: + secs: 1733883032 + nsecs: 206078529 +frame_id: '' and grasped_object_mask header seq: 40818 +stamp: + secs: 1733883031 + nsecs: 985645294 +frame_id: "camera_color_optical_frame" +[INFO] [1733883032.406154]: received mask_obs with header seq: 683 +stamp: + secs: 1733883032 + nsecs: 406803131 +frame_id: '' and grasped_object_mask header seq: 40824 +stamp: + secs: 1733883032 + nsecs: 185257673 +frame_id: "camera_color_optical_frame" +[INFO] [1733883032.601464]: received mask_obs with header seq: 684 +stamp: + secs: 1733883032 + nsecs: 600969314 +frame_id: '' and grasped_object_mask header seq: 40830 +stamp: + secs: 1733883032 + nsecs: 384857893 +frame_id: "camera_color_optical_frame" +[INFO] [1733883032.805261]: received mask_obs with header seq: 685 +stamp: + secs: 1733883032 + nsecs: 805272340 +frame_id: '' and grasped_object_mask header seq: 40836 +stamp: + secs: 1733883032 + nsecs: 584446192 +frame_id: "camera_color_optical_frame" +[INFO] [1733883033.006902]: received mask_obs with header seq: 686 +stamp: + secs: 1733883033 + nsecs: 6402969 +frame_id: '' and grasped_object_mask header seq: 40842 +stamp: + secs: 1733883032 + nsecs: 784022808 +frame_id: "camera_color_optical_frame" +[INFO] [1733883033.205207]: received mask_obs with header seq: 687 +stamp: + secs: 1733883033 + nsecs: 206177711 +frame_id: '' and grasped_object_mask header seq: 40848 +stamp: + secs: 1733883032 + nsecs: 983588457 +frame_id: "camera_color_optical_frame" +[INFO] [1733883033.413912]: received mask_obs with header seq: 688 +stamp: + secs: 1733883033 + nsecs: 414592742 +frame_id: '' and grasped_object_mask header seq: 40854 +stamp: + secs: 1733883033 + nsecs: 183202505 +frame_id: "camera_color_optical_frame" +[INFO] [1733883033.611213]: received mask_obs with header seq: 689 +stamp: + secs: 1733883033 + nsecs: 612060070 +frame_id: '' and grasped_object_mask header seq: 40860 +stamp: + secs: 1733883033 + nsecs: 382804632 +frame_id: "camera_color_optical_frame" +[INFO] [1733883033.806334]: received mask_obs with header seq: 690 +stamp: + secs: 1733883033 + nsecs: 807222604 +frame_id: '' and grasped_object_mask header seq: 40866 +stamp: + secs: 1733883033 + nsecs: 582395315 +frame_id: "camera_color_optical_frame" +[INFO] [1733883034.004493]: received mask_obs with header seq: 691 +stamp: + secs: 1733883034 + nsecs: 5437374 +frame_id: '' and grasped_object_mask header seq: 40872 +stamp: + secs: 1733883033 + nsecs: 781974077 +frame_id: "camera_color_optical_frame" +[INFO] [1733883034.198444]: received mask_obs with header seq: 692 +stamp: + secs: 1733883034 + nsecs: 199244737 +frame_id: '' and grasped_object_mask header seq: 40878 +stamp: + secs: 1733883033 + nsecs: 981541157 +frame_id: "camera_color_optical_frame" +[INFO] [1733883034.406420]: received mask_obs with header seq: 693 +stamp: + secs: 1733883034 + nsecs: 407005786 +frame_id: '' and grasped_object_mask header seq: 40884 +stamp: + secs: 1733883034 + nsecs: 181159973 +frame_id: "camera_color_optical_frame" +[INFO] [1733883034.810015]: received mask_obs with header seq: 694 +stamp: + secs: 1733883034 + nsecs: 810850620 +frame_id: '' and grasped_object_mask header seq: 40890 +stamp: + secs: 1733883034 + nsecs: 380766153 +frame_id: "camera_color_optical_frame" +[INFO] [1733883034.998891]: received mask_obs with header seq: 695 +stamp: + secs: 1733883034 + nsecs: 999424219 +frame_id: '' and grasped_object_mask header seq: 40902 +stamp: + secs: 1733883034 + nsecs: 779944420 +frame_id: "camera_color_optical_frame" +[INFO] [1733883035.197100]: received mask_obs with header seq: 696 +stamp: + secs: 1733883035 + nsecs: 197110414 +frame_id: '' and grasped_object_mask header seq: 40908 +stamp: + secs: 1733883034 + nsecs: 979516029 +frame_id: "camera_color_optical_frame" +[INFO] [1733883035.399836]: received mask_obs with header seq: 697 +stamp: + secs: 1733883035 + nsecs: 400691032 +frame_id: '' and grasped_object_mask header seq: 40914 +stamp: + secs: 1733883035 + nsecs: 179130077 +frame_id: "camera_color_optical_frame" +[INFO] [1733883035.589254]: received mask_obs with header seq: 698 +stamp: + secs: 1733883035 + nsecs: 590167760 +frame_id: '' and grasped_object_mask header seq: 40919 +stamp: + secs: 1733883035 + nsecs: 345466614 +frame_id: "camera_color_optical_frame" +[INFO] [1733883035.782454]: received mask_obs with header seq: 699 +stamp: + secs: 1733883035 + nsecs: 783262729 +frame_id: '' and grasped_object_mask header seq: 40925 +stamp: + secs: 1733883035 + nsecs: 545058966 +frame_id: "camera_color_optical_frame" +[INFO] [1733883035.975050]: received mask_obs with header seq: 700 +stamp: + secs: 1733883035 + nsecs: 975878477 +frame_id: '' and grasped_object_mask header seq: 40931 +stamp: + secs: 1733883035 + nsecs: 744639874 +frame_id: "camera_color_optical_frame" +[INFO] [1733883036.175865]: received mask_obs with header seq: 701 +stamp: + secs: 1733883036 + nsecs: 176237344 +frame_id: '' and grasped_object_mask header seq: 40937 +stamp: + secs: 1733883035 + nsecs: 944210052 +frame_id: "camera_color_optical_frame" +[INFO] [1733883036.383106]: received mask_obs with header seq: 702 +stamp: + secs: 1733883036 + nsecs: 383992671 +frame_id: '' and grasped_object_mask header seq: 40943 +stamp: + secs: 1733883036 + nsecs: 144964457 +frame_id: "camera_color_optical_frame" +[INFO] [1733883036.582577]: received mask_obs with header seq: 703 +stamp: + secs: 1733883036 + nsecs: 583411693 +frame_id: '' and grasped_object_mask header seq: 40949 +stamp: + secs: 1733883036 + nsecs: 346895456 +frame_id: "camera_color_optical_frame" +[INFO] [1733883036.773843]: received mask_obs with header seq: 704 +stamp: + secs: 1733883036 + nsecs: 774312019 +frame_id: '' and grasped_object_mask header seq: 40955 +stamp: + secs: 1733883036 + nsecs: 548901081 +frame_id: "camera_color_optical_frame" +[INFO] [1733883036.972461]: received mask_obs with header seq: 705 +stamp: + secs: 1733883036 + nsecs: 973179340 +frame_id: '' and grasped_object_mask header seq: 40961 +stamp: + secs: 1733883036 + nsecs: 750981092 +frame_id: "camera_color_optical_frame" +[INFO] [1733883037.170940]: received mask_obs with header seq: 706 +stamp: + secs: 1733883037 + nsecs: 171425580 +frame_id: '' and grasped_object_mask header seq: 40967 +stamp: + secs: 1733883036 + nsecs: 953136206 +frame_id: "camera_color_optical_frame" +[INFO] [1733883037.358173]: received mask_obs with header seq: 707 +stamp: + secs: 1733883037 + nsecs: 359020233 +frame_id: '' and grasped_object_mask header seq: 40972 +stamp: + secs: 1733883037 + nsecs: 120709419 +frame_id: "camera_color_optical_frame" +[INFO] [1733883037.552763]: received mask_obs with header seq: 708 +stamp: + secs: 1733883037 + nsecs: 553642511 +frame_id: '' and grasped_object_mask header seq: 40978 +stamp: + secs: 1733883037 + nsecs: 320096254 +frame_id: "camera_color_optical_frame" +[INFO] [1733883037.762185]: received mask_obs with header seq: 709 +stamp: + secs: 1733883037 + nsecs: 762813329 +frame_id: '' and grasped_object_mask header seq: 40984 +stamp: + secs: 1733883037 + nsecs: 519459724 +frame_id: "camera_color_optical_frame" +[INFO] [1733883037.956080]: received mask_obs with header seq: 710 +stamp: + secs: 1733883037 + nsecs: 956974506 +frame_id: '' and grasped_object_mask header seq: 40990 +stamp: + secs: 1733883037 + nsecs: 718800068 +frame_id: "camera_color_optical_frame" +[INFO] [1733883038.149457]: received mask_obs with header seq: 711 +stamp: + secs: 1733883038 + nsecs: 150292158 +frame_id: '' and grasped_object_mask header seq: 40996 +stamp: + secs: 1733883037 + nsecs: 918118000 +frame_id: "camera_color_optical_frame" +[INFO] [1733883038.345892]: received mask_obs with header seq: 712 +stamp: + secs: 1733883038 + nsecs: 346757888 +frame_id: '' and grasped_object_mask header seq: 41002 +stamp: + secs: 1733883038 + nsecs: 117870331 +frame_id: "camera_color_optical_frame" +[INFO] [1733883038.543372]: received mask_obs with header seq: 713 +stamp: + secs: 1733883038 + nsecs: 544145345 +frame_id: '' and grasped_object_mask header seq: 41008 +stamp: + secs: 1733883038 + nsecs: 318583250 +frame_id: "camera_color_optical_frame" +[INFO] [1733883038.743735]: received mask_obs with header seq: 714 +stamp: + secs: 1733883038 + nsecs: 744630575 +frame_id: '' and grasped_object_mask header seq: 41014 +stamp: + secs: 1733883038 + nsecs: 519372463 +frame_id: "camera_color_optical_frame" +[INFO] [1733883038.942803]: received mask_obs with header seq: 715 +stamp: + secs: 1733883038 + nsecs: 943788528 +frame_id: '' and grasped_object_mask header seq: 41020 +stamp: + secs: 1733883038 + nsecs: 720237732 +frame_id: "camera_color_optical_frame" +[INFO] [1733883039.131429]: received mask_obs with header seq: 716 +stamp: + secs: 1733883039 + nsecs: 132328271 +frame_id: '' and grasped_object_mask header seq: 41026 +stamp: + secs: 1733883038 + nsecs: 921179533 +frame_id: "camera_color_optical_frame" +[INFO] [1733883039.333827]: received mask_obs with header seq: 717 +stamp: + secs: 1733883039 + nsecs: 334663629 +frame_id: '' and grasped_object_mask header seq: 41032 +stamp: + secs: 1733883039 + nsecs: 121725798 +frame_id: "camera_color_optical_frame" +[INFO] [1733883039.528027]: received mask_obs with header seq: 718 +stamp: + secs: 1733883039 + nsecs: 528957128 +frame_id: '' and grasped_object_mask header seq: 41038 +stamp: + secs: 1733883039 + nsecs: 321343660 +frame_id: "camera_color_optical_frame" +[INFO] [1733883039.733516]: received mask_obs with header seq: 719 +stamp: + secs: 1733883039 + nsecs: 734317302 +frame_id: '' and grasped_object_mask header seq: 41043 +stamp: + secs: 1733883039 + nsecs: 487682819 +frame_id: "camera_color_optical_frame" +[INFO] [1733883039.933517]: received mask_obs with header seq: 720 +stamp: + secs: 1733883039 + nsecs: 934033870 +frame_id: '' and grasped_object_mask header seq: 41050 +stamp: + secs: 1733883039 + nsecs: 720545769 +frame_id: "camera_color_optical_frame" +[INFO] [1733883040.136336]: received mask_obs with header seq: 721 +stamp: + secs: 1733883040 + nsecs: 137259006 +frame_id: '' and grasped_object_mask header seq: 41056 +stamp: + secs: 1733883039 + nsecs: 920129776 +frame_id: "camera_color_optical_frame" +[INFO] [1733883040.329910]: received mask_obs with header seq: 722 +stamp: + secs: 1733883040 + nsecs: 330784559 +frame_id: '' and grasped_object_mask header seq: 41062 +stamp: + secs: 1733883040 + nsecs: 119718790 +frame_id: "camera_color_optical_frame" +[INFO] [1733883040.531031]: received mask_obs with header seq: 723 +stamp: + secs: 1733883040 + nsecs: 531894683 +frame_id: '' and grasped_object_mask header seq: 41068 +stamp: + secs: 1733883040 + nsecs: 319329023 +frame_id: "camera_color_optical_frame" +[INFO] [1733883040.722666]: received mask_obs with header seq: 724 +stamp: + secs: 1733883040 + nsecs: 723557710 +frame_id: '' and grasped_object_mask header seq: 41073 +stamp: + secs: 1733883040 + nsecs: 485660553 +frame_id: "camera_color_optical_frame" +[INFO] [1733883040.937450]: received mask_obs with header seq: 725 +stamp: + secs: 1733883040 + nsecs: 937803506 +frame_id: '' and grasped_object_mask header seq: 41079 +stamp: + secs: 1733883040 + nsecs: 685248852 +frame_id: "camera_color_optical_frame" +[INFO] [1733883041.141844]: received mask_obs with header seq: 726 +stamp: + secs: 1733883041 + nsecs: 142750501 +frame_id: '' and grasped_object_mask header seq: 41086 +stamp: + secs: 1733883040 + nsecs: 918087244 +frame_id: "camera_color_optical_frame" +[INFO] [1733883041.326676]: received mask_obs with header seq: 727 +stamp: + secs: 1733883041 + nsecs: 327246189 +frame_id: '' and grasped_object_mask header seq: 41092 +stamp: + secs: 1733883041 + nsecs: 117670298 +frame_id: "camera_color_optical_frame" +[INFO] [1733883041.511541]: received mask_obs with header seq: 728 +stamp: + secs: 1733883041 + nsecs: 512199640 +frame_id: '' and grasped_object_mask header seq: 41097 +stamp: + secs: 1733883041 + nsecs: 284015894 +frame_id: "camera_color_optical_frame" +[INFO] [1733883041.706550]: received mask_obs with header seq: 729 +stamp: + secs: 1733883041 + nsecs: 707083940 +frame_id: '' and grasped_object_mask header seq: 41103 +stamp: + secs: 1733883041 + nsecs: 483619213 +frame_id: "camera_color_optical_frame" +[INFO] [1733883041.911108]: received mask_obs with header seq: 730 +stamp: + secs: 1733883041 + nsecs: 912013053 +frame_id: '' and grasped_object_mask header seq: 41109 +stamp: + secs: 1733883041 + nsecs: 683210850 +frame_id: "camera_color_optical_frame" +[INFO] [1733883042.111341]: received mask_obs with header seq: 731 +stamp: + secs: 1733883042 + nsecs: 112192153 +frame_id: '' and grasped_object_mask header seq: 41115 +stamp: + secs: 1733883041 + nsecs: 882791042 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -246710311 for nearest: 1733883041835659017 - target: 1733883042082369328 at idx 203 +[INFO] [1733883042.300270]: received mask_obs with header seq: 732 +stamp: + secs: 1733883042 + nsecs: 301242351 +frame_id: '' and grasped_object_mask header seq: 41121 +stamp: + secs: 1733883042 + nsecs: 82369328 +frame_id: "camera_color_optical_frame" +[INFO] [1733883042.505503]: received mask_obs with header seq: 733 +stamp: + secs: 1733883042 + nsecs: 506350994 +frame_id: '' and grasped_object_mask header seq: 41127 +stamp: + secs: 1733883042 + nsecs: 281985998 +frame_id: "camera_color_optical_frame" +[INFO] [1733883042.692483]: received mask_obs with header seq: 734 +stamp: + secs: 1733883042 + nsecs: 693349838 +frame_id: '' and grasped_object_mask header seq: 41133 +stamp: + secs: 1733883042 + nsecs: 481589794 +frame_id: "camera_color_optical_frame" +[INFO] [1733883042.895209]: received mask_obs with header seq: 735 +stamp: + secs: 1733883042 + nsecs: 895925283 +frame_id: '' and grasped_object_mask header seq: 41139 +stamp: + secs: 1733883042 + nsecs: 681182384 +frame_id: "camera_color_optical_frame" +[INFO] [1733883043.091529]: received mask_obs with header seq: 736 +stamp: + secs: 1733883043 + nsecs: 92341899 +frame_id: '' and grasped_object_mask header seq: 41145 +stamp: + secs: 1733883042 + nsecs: 880763292 +frame_id: "camera_color_optical_frame" +[INFO] [1733883043.306891]: received mask_obs with header seq: 737 +stamp: + secs: 1733883043 + nsecs: 307593584 +frame_id: '' and grasped_object_mask header seq: 41151 +stamp: + secs: 1733883043 + nsecs: 80342531 +frame_id: "camera_color_optical_frame" +[INFO] [1733883043.498719]: received mask_obs with header seq: 738 +stamp: + secs: 1733883043 + nsecs: 499245643 +frame_id: '' and grasped_object_mask header seq: 41157 +stamp: + secs: 1733883043 + nsecs: 279960155 +frame_id: "camera_color_optical_frame" +[INFO] [1733883043.708814]: received mask_obs with header seq: 739 +stamp: + secs: 1733883043 + nsecs: 709574460 +frame_id: '' and grasped_object_mask header seq: 41162 +stamp: + secs: 1733883043 + nsecs: 446300030 +frame_id: "camera_color_optical_frame" +[INFO] [1733883043.901069]: received mask_obs with header seq: 740 +stamp: + secs: 1733883043 + nsecs: 901894807 +frame_id: '' and grasped_object_mask header seq: 41169 +stamp: + secs: 1733883043 + nsecs: 679162502 +frame_id: "camera_color_optical_frame" +[INFO] [1733883044.096456]: received mask_obs with header seq: 741 +stamp: + secs: 1733883044 + nsecs: 97338199 +frame_id: '' and grasped_object_mask header seq: 41175 +stamp: + secs: 1733883043 + nsecs: 878746033 +frame_id: "camera_color_optical_frame" +[INFO] [1733883044.298318]: received mask_obs with header seq: 742 +stamp: + secs: 1733883044 + nsecs: 298749208 +frame_id: '' and grasped_object_mask header seq: 41181 +stamp: + secs: 1733883044 + nsecs: 78327656 +frame_id: "camera_color_optical_frame" +[INFO] [1733883044.490426]: received mask_obs with header seq: 743 +stamp: + secs: 1733883044 + nsecs: 490856409 +frame_id: '' and grasped_object_mask header seq: 41187 +stamp: + secs: 1733883044 + nsecs: 277944088 +frame_id: "camera_color_optical_frame" +[INFO] [1733883044.694947]: received mask_obs with header seq: 744 +stamp: + secs: 1733883044 + nsecs: 695871591 +frame_id: '' and grasped_object_mask header seq: 41192 +stamp: + secs: 1733883044 + nsecs: 444282770 +frame_id: "camera_color_optical_frame" +[INFO] [1733883044.900694]: received mask_obs with header seq: 745 +stamp: + secs: 1733883044 + nsecs: 900617122 +frame_id: '' and grasped_object_mask header seq: 41198 +stamp: + secs: 1733883044 + nsecs: 643878222 +frame_id: "camera_color_optical_frame" +[INFO] [1733883045.094872]: received mask_obs with header seq: 746 +stamp: + secs: 1733883045 + nsecs: 94925165 +frame_id: '' and grasped_object_mask header seq: 41205 +stamp: + secs: 1733883044 + nsecs: 876725674 +frame_id: "camera_color_optical_frame" +[INFO] [1733883045.293305]: received mask_obs with header seq: 747 +stamp: + secs: 1733883045 + nsecs: 293159484 +frame_id: '' and grasped_object_mask header seq: 41211 +stamp: + secs: 1733883045 + nsecs: 76304913 +frame_id: "camera_color_optical_frame" +[INFO] [1733883045.488447]: received mask_obs with header seq: 748 +stamp: + secs: 1733883045 + nsecs: 488064050 +frame_id: '' and grasped_object_mask header seq: 41216 +stamp: + secs: 1733883045 + nsecs: 242650032 +frame_id: "camera_color_optical_frame" +[INFO] [1733883045.691041]: received mask_obs with header seq: 749 +stamp: + secs: 1733883045 + nsecs: 691947221 +frame_id: '' and grasped_object_mask header seq: 41223 +stamp: + secs: 1733883045 + nsecs: 475518703 +frame_id: "camera_color_optical_frame" +[INFO] [1733883045.891014]: received mask_obs with header seq: 750 +stamp: + secs: 1733883045 + nsecs: 891385555 +frame_id: '' and grasped_object_mask header seq: 41229 +stamp: + secs: 1733883045 + nsecs: 675108194 +frame_id: "camera_color_optical_frame" +[INFO] [1733883046.089338]: received mask_obs with header seq: 751 +stamp: + secs: 1733883046 + nsecs: 89556455 +frame_id: '' and grasped_object_mask header seq: 41234 +stamp: + secs: 1733883045 + nsecs: 841423988 +frame_id: "camera_color_optical_frame" +[INFO] [1733883046.287296]: received mask_obs with header seq: 752 +stamp: + secs: 1733883046 + nsecs: 287266969 +frame_id: '' and grasped_object_mask header seq: 41240 +stamp: + secs: 1733883046 + nsecs: 40992260 +frame_id: "camera_color_optical_frame" +[INFO] [1733883046.478949]: received mask_obs with header seq: 753 +stamp: + secs: 1733883046 + nsecs: 479885816 +frame_id: '' and grasped_object_mask header seq: 41246 +stamp: + secs: 1733883046 + nsecs: 240609646 +frame_id: "camera_color_optical_frame" +[INFO] [1733883046.673161]: received mask_obs with header seq: 754 +stamp: + secs: 1733883046 + nsecs: 673880815 +frame_id: '' and grasped_object_mask header seq: 41252 +stamp: + secs: 1733883046 + nsecs: 440216541 +frame_id: "camera_color_optical_frame" +[INFO] [1733883046.874100]: received mask_obs with header seq: 755 +stamp: + secs: 1733883046 + nsecs: 874928951 +frame_id: '' and grasped_object_mask header seq: 41258 +stamp: + secs: 1733883046 + nsecs: 639810801 +frame_id: "camera_color_optical_frame" +[INFO] [1733883047.074032]: received mask_obs with header seq: 756 +stamp: + secs: 1733883047 + nsecs: 74733972 +frame_id: '' and grasped_object_mask header seq: 41264 +stamp: + secs: 1733883046 + nsecs: 839393616 +frame_id: "camera_color_optical_frame" +[INFO] [1733883047.262760]: received mask_obs with header seq: 757 +stamp: + secs: 1733883047 + nsecs: 263715982 +frame_id: '' and grasped_object_mask header seq: 41270 +stamp: + secs: 1733883047 + nsecs: 38964748 +frame_id: "camera_color_optical_frame" +[INFO] [1733883047.462683]: received mask_obs with header seq: 758 +stamp: + secs: 1733883047 + nsecs: 463361978 +frame_id: '' and grasped_object_mask header seq: 41276 +stamp: + secs: 1733883047 + nsecs: 238574028 +frame_id: "camera_color_optical_frame" +[INFO] [1733883047.654940]: received mask_obs with header seq: 759 +stamp: + secs: 1733883047 + nsecs: 655624389 +frame_id: '' and grasped_object_mask header seq: 41282 +stamp: + secs: 1733883047 + nsecs: 438171387 +frame_id: "camera_color_optical_frame" +[INFO] [1733883047.846232]: received mask_obs with header seq: 760 +stamp: + secs: 1733883047 + nsecs: 846917629 +frame_id: '' and grasped_object_mask header seq: 41287 +stamp: + secs: 1733883047 + nsecs: 604493856 +frame_id: "camera_color_optical_frame" +[INFO] [1733883048.060377]: received mask_obs with header seq: 761 +stamp: + secs: 1733883048 + nsecs: 61100482 +frame_id: '' and grasped_object_mask header seq: 41293 +stamp: + secs: 1733883047 + nsecs: 804069519 +frame_id: "camera_color_optical_frame" +[INFO] [1733883048.266974]: received mask_obs with header seq: 762 +stamp: + secs: 1733883048 + nsecs: 267392396 +frame_id: '' and grasped_object_mask header seq: 41300 +stamp: + secs: 1733883048 + nsecs: 36893368 +frame_id: "camera_color_optical_frame" +[INFO] [1733883048.465165]: received mask_obs with header seq: 763 +stamp: + secs: 1733883048 + nsecs: 465203762 +frame_id: '' and grasped_object_mask header seq: 41306 +stamp: + secs: 1733883048 + nsecs: 236512184 +frame_id: "camera_color_optical_frame" +[INFO] [1733883048.662735]: received mask_obs with header seq: 764 +stamp: + secs: 1733883048 + nsecs: 662102222 +frame_id: '' and grasped_object_mask header seq: 41312 +stamp: + secs: 1733883048 + nsecs: 436119556 +frame_id: "camera_color_optical_frame" +For topic gripper_width: timestamp difference is -100056017 for nearest: 1733883048535659468 - target: 1733883048635715485 at idx 249 +[INFO] [1733883048.859946]: received mask_obs with header seq: 765 +stamp: + secs: 1733883048 + nsecs: 859627723 +frame_id: '' and grasped_object_mask header seq: 41318 +stamp: + secs: 1733883048 + nsecs: 635715485 +frame_id: "camera_color_optical_frame" +[INFO] [1733883049.024784]: received mask_obs with header seq: 766 +stamp: + secs: 1733883049 + nsecs: 24731636 +frame_id: '' and grasped_object_mask header seq: 41323 +stamp: + secs: 1733883048 + nsecs: 802037001 +frame_id: "camera_color_optical_frame" +[INFO] [1733883049.219549]: received mask_obs with header seq: 767 +stamp: + secs: 1733883049 + nsecs: 220157146 +frame_id: '' and grasped_object_mask header seq: 41329 +stamp: + secs: 1733883049 + nsecs: 1611948 +frame_id: "camera_color_optical_frame" +[INFO] [1733883049.425859]: received mask_obs with header seq: 768 +stamp: + secs: 1733883049 + nsecs: 426707267 +frame_id: '' and grasped_object_mask header seq: 41335 +stamp: + secs: 1733883049 + nsecs: 201227188 +frame_id: "camera_color_optical_frame" +[INFO] [1733883049.609217]: received mask_obs with header seq: 769 +stamp: + secs: 1733883049 + nsecs: 610003232 +frame_id: '' and grasped_object_mask header seq: 41340 +stamp: + secs: 1733883049 + nsecs: 367573738 +frame_id: "camera_color_optical_frame" +[INFO] [1733883049.802361]: received mask_obs with header seq: 770 +stamp: + secs: 1733883049 + nsecs: 802881002 +frame_id: '' and grasped_object_mask header seq: 41346 +stamp: + secs: 1733883049 + nsecs: 567178726 +frame_id: "camera_color_optical_frame" +[INFO] [1733883050.003291]: received mask_obs with header seq: 771 +stamp: + secs: 1733883050 + nsecs: 4029989 +frame_id: '' and grasped_object_mask header seq: 41352 +stamp: + secs: 1733883049 + nsecs: 766772270 +frame_id: "camera_color_optical_frame" +[INFO] [1733883050.202075]: received mask_obs with header seq: 772 +stamp: + secs: 1733883050 + nsecs: 202923297 +frame_id: '' and grasped_object_mask header seq: 41358 +stamp: + secs: 1733883049 + nsecs: 966354609 +frame_id: "camera_color_optical_frame" +[INFO] [1733883050.402999]: received mask_obs with header seq: 773 +stamp: + secs: 1733883050 + nsecs: 403917789 +frame_id: '' and grasped_object_mask header seq: 41364 +stamp: + secs: 1733883050 + nsecs: 165958405 +frame_id: "camera_color_optical_frame" +[INFO] [1733883050.610002]: received mask_obs with header seq: 774 +stamp: + secs: 1733883050 + nsecs: 610520601 +frame_id: '' and grasped_object_mask header seq: 41370 +stamp: + secs: 1733883050 + nsecs: 365566015 +frame_id: "camera_color_optical_frame" +[INFO] [1733883050.808163]: received mask_obs with header seq: 775 +stamp: + secs: 1733883050 + nsecs: 809062004 +frame_id: '' and grasped_object_mask header seq: 41376 +stamp: + secs: 1733883050 + nsecs: 565161467 +frame_id: "camera_color_optical_frame" +[INFO] [1733883051.002382]: received mask_obs with header seq: 776 +stamp: + secs: 1733883051 + nsecs: 2823114 +frame_id: '' and grasped_object_mask header seq: 41382 +stamp: + secs: 1733883050 + nsecs: 764745474 +frame_id: "camera_color_optical_frame" +[INFO] [1733883051.212790]: received mask_obs with header seq: 777 +stamp: + secs: 1733883051 + nsecs: 213707923 +frame_id: '' and grasped_object_mask header seq: 41388 +stamp: + secs: 1733883050 + nsecs: 964317799 +frame_id: "camera_color_optical_frame" +[INFO] [1733883051.414550]: received mask_obs with header seq: 778 +stamp: + secs: 1733883051 + nsecs: 415505170 +frame_id: '' and grasped_object_mask header seq: 41394 +stamp: + secs: 1733883051 + nsecs: 163915157 +frame_id: "camera_color_optical_frame" +[INFO] [1733883051.618629]: received mask_obs with header seq: 779 +stamp: + secs: 1733883051 + nsecs: 619503259 +frame_id: '' and grasped_object_mask header seq: 41400 +stamp: + secs: 1733883051 + nsecs: 363519669 +frame_id: "camera_color_optical_frame" +[INFO] [1733883051.821563]: received mask_obs with header seq: 780 +stamp: + secs: 1733883051 + nsecs: 822328567 +frame_id: '' and grasped_object_mask header seq: 41406 +stamp: + secs: 1733883051 + nsecs: 563111782 +frame_id: "camera_color_optical_frame" +[INFO] [1733883052.019144]: received mask_obs with header seq: 781 +stamp: + secs: 1733883052 + nsecs: 18641471 +frame_id: '' and grasped_object_mask header seq: 41413 +stamp: + secs: 1733883051 + nsecs: 795953274 +frame_id: "camera_color_optical_frame" +[INFO] [1733883052.212928]: received mask_obs with header seq: 782 +stamp: + secs: 1733883052 + nsecs: 213798999 +frame_id: '' and grasped_object_mask header seq: 41418 +stamp: + secs: 1733883051 + nsecs: 962259769 +frame_id: "camera_color_optical_frame" +[INFO] [1733883052.404430]: received mask_obs with header seq: 783 +stamp: + secs: 1733883052 + nsecs: 405048847 +frame_id: '' and grasped_object_mask header seq: 41424 +stamp: + secs: 1733883052 + nsecs: 162691593 +frame_id: "camera_color_optical_frame" +[INFO] [1733883052.598324]: received mask_obs with header seq: 784 +stamp: + secs: 1733883052 + nsecs: 599291324 +frame_id: '' and grasped_object_mask header seq: 41430 +stamp: + secs: 1733883052 + nsecs: 363631248 +frame_id: "camera_color_optical_frame" +[INFO] [1733883052.805919]: received mask_obs with header seq: 785 +stamp: + secs: 1733883052 + nsecs: 806646108 +frame_id: '' and grasped_object_mask header seq: 41436 +stamp: + secs: 1733883052 + nsecs: 564657688 +frame_id: "camera_color_optical_frame" +[INFO] [1733883053.017940]: received mask_obs with header seq: 786 +stamp: + secs: 1733883053 + nsecs: 18372774 +frame_id: '' and grasped_object_mask header seq: 41442 +stamp: + secs: 1733883052 + nsecs: 765771389 +frame_id: "camera_color_optical_frame" +[INFO] [1733883053.215296]: received mask_obs with header seq: 787 +stamp: + secs: 1733883053 + nsecs: 215991735 +frame_id: '' and grasped_object_mask header seq: 41449 +stamp: + secs: 1733883053 + nsecs: 511885 +frame_id: "camera_color_optical_frame" +[INFO] [1733883053.421379]: received mask_obs with header seq: 788 +stamp: + secs: 1733883053 + nsecs: 422115087 +frame_id: '' and grasped_object_mask header seq: 41454 +stamp: + secs: 1733883053 + nsecs: 167284012 +frame_id: "camera_color_optical_frame" +[INFO] [1733883053.604455]: received mask_obs with header seq: 789 +stamp: + secs: 1733883053 + nsecs: 605422019 +frame_id: '' and grasped_object_mask header seq: 41460 +stamp: + secs: 1733883053 + nsecs: 367125273 +frame_id: "camera_color_optical_frame" +[INFO] [1733883053.800578]: received mask_obs with header seq: 790 +stamp: + secs: 1733883053 + nsecs: 801460266 +frame_id: '' and grasped_object_mask header seq: 41466 +stamp: + secs: 1733883053 + nsecs: 566966772 +frame_id: "camera_color_optical_frame" +[INFO] [1733883053.853217]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1733883053.998865]: received mask_obs with header seq: 791 +stamp: + secs: 1733883053 + nsecs: 999423027 +frame_id: '' and grasped_object_mask header seq: 41472 +stamp: + secs: 1733883053 + nsecs: 766808510 +frame_id: "camera_color_optical_frame" +[INFO] [1733883054.659320]: Stored episode 4. +[INFO] [1733883054.662071]: Saving video... +[INFO] [1733883054.998219]: Video saved! +Starting episode 4 +[INFO] [1733883055.049076]: resetting environment +[INFO] [1733883055.062911]: cleared current plan +[INFO] [1733883055.064263]: moving to home +[INFO] [1733883057.871810]: reached home +[INFO] [1733883057.874172]: reset action history +[INFO] [1733883060.739273]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1733883063.407213]: resetting environment +[INFO] [1733883063.423720]: cleared current plan +[INFO] [1733883063.427989]: moving to home +[INFO] [1733883064.435215]: reached home +[INFO] [1733883064.436928]: reset action history +[INFO] [1733883067.163568]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/episode_rosbags/episode_4_2024-12-10-21-11-07.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-600897.log +started roslaunch server http://158.130.50.37:37615/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [601740] +process[print_text-14]: started with pid [601741] +process[rosbag_record-15]: started with pid [601765] +started bagging! +[INFO] [1733883067.857057]: received mask_obs with header seq: 792 +stamp: + secs: 1733883067 + nsecs: 857721805 +frame_id: '' and grasped_object_mask header seq: 41889 +stamp: + secs: 1733883067 + nsecs: 655425787 +frame_id: "camera_color_optical_frame" +[INFO] [1733883068.059242]: received mask_obs with header seq: 793 +stamp: + secs: 1733883068 + nsecs: 60029268 +frame_id: '' and grasped_object_mask header seq: 41895 +stamp: + secs: 1733883067 + nsecs: 855261564 +frame_id: "camera_color_optical_frame" +[INFO] [1733883068.261637]: received mask_obs with header seq: 794 +stamp: + secs: 1733883068 + nsecs: 262381315 +frame_id: '' and grasped_object_mask header seq: 41901 +stamp: + secs: 1733883068 + nsecs: 55098057 +frame_id: "camera_color_optical_frame" +[INFO] [1733883068.463790]: received mask_obs with header seq: 795 +stamp: + secs: 1733883068 + nsecs: 464706659 +frame_id: '' and grasped_object_mask header seq: 41907 +stamp: + secs: 1733883068 + nsecs: 254940033 +frame_id: "camera_color_optical_frame" +[INFO] [1733883068.667338]: received mask_obs with header seq: 796 +stamp: + secs: 1733883068 + nsecs: 667449235 +frame_id: '' and grasped_object_mask header seq: 41913 +stamp: + secs: 1733883068 + nsecs: 454782009 +frame_id: "camera_color_optical_frame" +[INFO] [1733883068.887617]: received mask_obs with header seq: 797 +stamp: + secs: 1733883068 + nsecs: 888394832 +frame_id: '' and grasped_object_mask header seq: 41919 +stamp: + secs: 1733883068 + nsecs: 654624224 +frame_id: "camera_color_optical_frame" +[INFO] [1733883069.090440]: received mask_obs with header seq: 798 +stamp: + secs: 1733883069 + nsecs: 91318130 +frame_id: '' and grasped_object_mask header seq: 41925 +stamp: + secs: 1733883068 + nsecs: 854466438 +frame_id: "camera_color_optical_frame" +[INFO] [1733883069.276004]: received mask_obs with header seq: 799 +stamp: + secs: 1733883069 + nsecs: 276580333 +frame_id: '' and grasped_object_mask header seq: 41931 +stamp: + secs: 1733883069 + nsecs: 54309368 +frame_id: "camera_color_optical_frame" +[INFO] [1733883069.478409]: received mask_obs with header seq: 800 +stamp: + secs: 1733883069 + nsecs: 479287862 +frame_id: '' and grasped_object_mask header seq: 41937 +stamp: + secs: 1733883069 + nsecs: 254156590 +frame_id: "camera_color_optical_frame" +[INFO] [1733883069.685149]: received mask_obs with header seq: 801 +stamp: + secs: 1733883069 + nsecs: 686037778 +frame_id: '' and grasped_object_mask header seq: 41943 +stamp: + secs: 1733883069 + nsecs: 454002857 +frame_id: "camera_color_optical_frame" +[INFO] [1733883069.887985]: received mask_obs with header seq: 802 +stamp: + secs: 1733883069 + nsecs: 888587713 +frame_id: '' and grasped_object_mask header seq: 41949 +stamp: + secs: 1733883069 + nsecs: 653850079 +frame_id: "camera_color_optical_frame" +[INFO] [1733883070.081005]: received mask_obs with header seq: 803 +stamp: + secs: 1733883070 + nsecs: 81546306 +frame_id: '' and grasped_object_mask header seq: 41955 +stamp: + secs: 1733883069 + nsecs: 853697300 +frame_id: "camera_color_optical_frame" +[INFO] [1733883070.279178]: received mask_obs with header seq: 804 +stamp: + secs: 1733883070 + nsecs: 280030488 +frame_id: '' and grasped_object_mask header seq: 41961 +stamp: + secs: 1733883070 + nsecs: 53545237 +frame_id: "camera_color_optical_frame" +[INFO] [1733883070.477928]: received mask_obs with header seq: 805 +stamp: + secs: 1733883070 + nsecs: 478755235 +frame_id: '' and grasped_object_mask header seq: 41967 +stamp: + secs: 1733883070 + nsecs: 253388643 +frame_id: "camera_color_optical_frame" +[INFO] [1733883070.686765]: received mask_obs with header seq: 806 +stamp: + secs: 1733883070 + nsecs: 687154531 +frame_id: '' and grasped_object_mask header seq: 41973 +stamp: + secs: 1733883070 + nsecs: 453232527 +frame_id: "camera_color_optical_frame" +[INFO] [1733883070.879082]: received mask_obs with header seq: 807 +stamp: + secs: 1733883070 + nsecs: 880019664 +frame_id: '' and grasped_object_mask header seq: 41979 +stamp: + secs: 1733883070 + nsecs: 653077602 +frame_id: "camera_color_optical_frame" +[INFO] [1733883071.078130]: received mask_obs with header seq: 808 +stamp: + secs: 1733883071 + nsecs: 79074382 +frame_id: '' and grasped_object_mask header seq: 41985 +stamp: + secs: 1733883070 + nsecs: 852922201 +frame_id: "camera_color_optical_frame" +[INFO] [1733883071.278259]: received mask_obs with header seq: 809 +stamp: + secs: 1733883071 + nsecs: 278701782 +frame_id: '' and grasped_object_mask header seq: 41991 +stamp: + secs: 1733883071 + nsecs: 52766800 +frame_id: "camera_color_optical_frame" +[INFO] [1733883071.473873]: received mask_obs with header seq: 810 +stamp: + secs: 1733883071 + nsecs: 474799394 +frame_id: '' and grasped_object_mask header seq: 41997 +stamp: + secs: 1733883071 + nsecs: 252612114 +frame_id: "camera_color_optical_frame" +[INFO] [1733883071.675026]: received mask_obs with header seq: 811 +stamp: + secs: 1733883071 + nsecs: 674627542 +frame_id: '' and grasped_object_mask header seq: 42003 +stamp: + secs: 1733883071 + nsecs: 452457666 +frame_id: "camera_color_optical_frame" +[INFO] [1733883071.894456]: received mask_obs with header seq: 812 +stamp: + secs: 1733883071 + nsecs: 895400285 +frame_id: '' and grasped_object_mask header seq: 42009 +stamp: + secs: 1733883071 + nsecs: 652303696 +frame_id: "camera_color_optical_frame" +[INFO] [1733883072.096142]: received mask_obs with header seq: 813 +stamp: + secs: 1733883072 + nsecs: 97013711 +frame_id: '' and grasped_object_mask header seq: 42015 +stamp: + secs: 1733883071 + nsecs: 852151155 +frame_id: "camera_color_optical_frame" +[INFO] [1733883072.301890]: received mask_obs with header seq: 814 +stamp: + secs: 1733883072 + nsecs: 302713394 +frame_id: '' and grasped_object_mask header seq: 42021 +stamp: + secs: 1733883072 + nsecs: 51998138 +frame_id: "camera_color_optical_frame" +[INFO] [1733883072.496067]: received mask_obs with header seq: 815 +stamp: + secs: 1733883072 + nsecs: 496965885 +frame_id: '' and grasped_object_mask header seq: 42027 +stamp: + secs: 1733883072 + nsecs: 251833200 +frame_id: "camera_color_optical_frame" +[INFO] [1733883072.696809]: received mask_obs with header seq: 816 +stamp: + secs: 1733883072 + nsecs: 697611570 +frame_id: '' and grasped_object_mask header seq: 42033 +stamp: + secs: 1733883072 + nsecs: 451667786 +frame_id: "camera_color_optical_frame" +[INFO] [1733883072.880915]: received mask_obs with header seq: 817 +stamp: + secs: 1733883072 + nsecs: 881675004 +frame_id: '' and grasped_object_mask header seq: 42039 +stamp: + secs: 1733883072 + nsecs: 651501894 +frame_id: "camera_color_optical_frame" +[INFO] [1733883073.074244]: received mask_obs with header seq: 818 +stamp: + secs: 1733883073 + nsecs: 74931621 +frame_id: '' and grasped_object_mask header seq: 42045 +stamp: + secs: 1733883072 + nsecs: 851336002 +frame_id: "camera_color_optical_frame" +[INFO] [1733883073.285005]: received mask_obs with header seq: 819 +stamp: + secs: 1733883073 + nsecs: 285944223 +frame_id: '' and grasped_object_mask header seq: 42051 +stamp: + secs: 1733883073 + nsecs: 51170826 +frame_id: "camera_color_optical_frame" +[INFO] [1733883073.484383]: received mask_obs with header seq: 820 +stamp: + secs: 1733883073 + nsecs: 485112428 +frame_id: '' and grasped_object_mask header seq: 42057 +stamp: + secs: 1733883073 + nsecs: 251008987 +frame_id: "camera_color_optical_frame" +[INFO] [1733883073.683152]: received mask_obs with header seq: 821 +stamp: + secs: 1733883073 + nsecs: 684057235 +frame_id: '' and grasped_object_mask header seq: 42063 +stamp: + secs: 1733883073 + nsecs: 450847387 +frame_id: "camera_color_optical_frame" +[INFO] [1733883073.877548]: received mask_obs with header seq: 822 +stamp: + secs: 1733883073 + nsecs: 877992153 +frame_id: '' and grasped_object_mask header seq: 42069 +stamp: + secs: 1733883073 + nsecs: 650685787 +frame_id: "camera_color_optical_frame" +[INFO] [1733883074.089243]: received mask_obs with header seq: 823 +stamp: + secs: 1733883074 + nsecs: 89946269 +frame_id: '' and grasped_object_mask header seq: 42075 +stamp: + secs: 1733883073 + nsecs: 850524187 +frame_id: "camera_color_optical_frame" +[INFO] [1733883074.289895]: received mask_obs with header seq: 824 +stamp: + secs: 1733883074 + nsecs: 290739774 +frame_id: '' and grasped_object_mask header seq: 42081 +stamp: + secs: 1733883074 + nsecs: 50362349 +frame_id: "camera_color_optical_frame" +[INFO] [1733883074.490880]: received mask_obs with header seq: 825 +stamp: + secs: 1733883074 + nsecs: 491274833 +frame_id: '' and grasped_object_mask header seq: 42087 +stamp: + secs: 1733883074 + nsecs: 250207663 +frame_id: "camera_color_optical_frame" +[INFO] [1733883074.702407]: received mask_obs with header seq: 826 +stamp: + secs: 1733883074 + nsecs: 703076124 +frame_id: '' and grasped_object_mask header seq: 42093 +stamp: + secs: 1733883074 + nsecs: 450052738 +frame_id: "camera_color_optical_frame" +[INFO] [1733883074.896667]: received mask_obs with header seq: 827 +stamp: + secs: 1733883074 + nsecs: 897521972 +frame_id: '' and grasped_object_mask header seq: 42099 +stamp: + secs: 1733883074 + nsecs: 649898052 +frame_id: "camera_color_optical_frame" +[INFO] [1733883075.080488]: received mask_obs with header seq: 828 +stamp: + secs: 1733883075 + nsecs: 81445693 +frame_id: '' and grasped_object_mask header seq: 42105 +stamp: + secs: 1733883074 + nsecs: 849743366 +frame_id: "camera_color_optical_frame" +[INFO] [1733883075.288503]: received mask_obs with header seq: 829 +stamp: + secs: 1733883075 + nsecs: 289255619 +frame_id: '' and grasped_object_mask header seq: 42111 +stamp: + secs: 1733883075 + nsecs: 49588919 +frame_id: "camera_color_optical_frame" +[INFO] [1733883075.479056]: received mask_obs with header seq: 830 +stamp: + secs: 1733883075 + nsecs: 479981422 +frame_id: '' and grasped_object_mask header seq: 42117 +stamp: + secs: 1733883075 + nsecs: 249431133 +frame_id: "camera_color_optical_frame" +[INFO] [1733883075.676364]: received mask_obs with header seq: 831 +stamp: + secs: 1733883075 + nsecs: 677111625 +frame_id: '' and grasped_object_mask header seq: 42123 +stamp: + secs: 1733883075 + nsecs: 449274063 +frame_id: "camera_color_optical_frame" +[INFO] [1733883075.887215]: received mask_obs with header seq: 832 +stamp: + secs: 1733883075 + nsecs: 888156175 +frame_id: '' and grasped_object_mask header seq: 42129 +stamp: + secs: 1733883075 + nsecs: 649116278 +frame_id: "camera_color_optical_frame" +[INFO] [1733883076.076431]: received mask_obs with header seq: 833 +stamp: + secs: 1733883076 + nsecs: 76924324 +frame_id: '' and grasped_object_mask header seq: 42135 +stamp: + secs: 1733883075 + nsecs: 848958492 +frame_id: "camera_color_optical_frame" +[INFO] [1733883076.286951]: received mask_obs with header seq: 834 +stamp: + secs: 1733883076 + nsecs: 287834644 +frame_id: '' and grasped_object_mask header seq: 42141 +stamp: + secs: 1733883076 + nsecs: 48800707 +frame_id: "camera_color_optical_frame" +[INFO] [1733883076.491155]: received mask_obs with header seq: 835 +stamp: + secs: 1733883076 + nsecs: 492028951 +frame_id: '' and grasped_object_mask header seq: 42147 +stamp: + secs: 1733883076 + nsecs: 248643398 +frame_id: "camera_color_optical_frame" +[INFO] [1733883076.689914]: received mask_obs with header seq: 836 +stamp: + secs: 1733883076 + nsecs: 690726757 +frame_id: '' and grasped_object_mask header seq: 42153 +stamp: + secs: 1733883076 + nsecs: 448486090 +frame_id: "camera_color_optical_frame" +[INFO] [1733883076.887678]: received mask_obs with header seq: 837 +stamp: + secs: 1733883076 + nsecs: 888211488 +frame_id: '' and grasped_object_mask header seq: 42159 +stamp: + secs: 1733883076 + nsecs: 648329973 +frame_id: "camera_color_optical_frame" +[INFO] [1733883077.077646]: received mask_obs with header seq: 838 +stamp: + secs: 1733883077 + nsecs: 78469991 +frame_id: '' and grasped_object_mask header seq: 42165 +stamp: + secs: 1733883076 + nsecs: 848173141 +frame_id: "camera_color_optical_frame" +[INFO] [1733883077.281827]: received mask_obs with header seq: 839 +stamp: + secs: 1733883077 + nsecs: 282119274 +frame_id: '' and grasped_object_mask header seq: 42171 +stamp: + secs: 1733883077 + nsecs: 48016071 +frame_id: "camera_color_optical_frame" +[INFO] [1733883077.475435]: received mask_obs with header seq: 840 +stamp: + secs: 1733883077 + nsecs: 476309299 +frame_id: '' and grasped_object_mask header seq: 42177 +stamp: + secs: 1733883077 + nsecs: 247863770 +frame_id: "camera_color_optical_frame" +[INFO] [1733883077.683994]: received mask_obs with header seq: 841 +stamp: + secs: 1733883077 + nsecs: 684493780 +frame_id: '' and grasped_object_mask header seq: 42183 +stamp: + secs: 1733883077 + nsecs: 447712421 +frame_id: "camera_color_optical_frame" +[INFO] [1733883077.881966]: received mask_obs with header seq: 842 +stamp: + secs: 1733883077 + nsecs: 882899761 +frame_id: '' and grasped_object_mask header seq: 42189 +stamp: + secs: 1733883077 + nsecs: 647561312 +frame_id: "camera_color_optical_frame" +[INFO] [1733883078.076070]: received mask_obs with header seq: 843 +stamp: + secs: 1733883078 + nsecs: 76845169 +frame_id: '' and grasped_object_mask header seq: 42195 +stamp: + secs: 1733883077 + nsecs: 847410679 +frame_id: "camera_color_optical_frame" +[INFO] [1733883078.276911]: received mask_obs with header seq: 844 +stamp: + secs: 1733883078 + nsecs: 277655839 +frame_id: '' and grasped_object_mask header seq: 42201 +stamp: + secs: 1733883078 + nsecs: 47261477 +frame_id: "camera_color_optical_frame" +[INFO] [1733883078.475921]: received mask_obs with header seq: 845 +stamp: + secs: 1733883078 + nsecs: 476553440 +frame_id: '' and grasped_object_mask header seq: 42207 +stamp: + secs: 1733883078 + nsecs: 247102022 +frame_id: "camera_color_optical_frame" +[INFO] [1733883078.683293]: received mask_obs with header seq: 846 +stamp: + secs: 1733883078 + nsecs: 683820962 +frame_id: '' and grasped_object_mask header seq: 42213 +stamp: + secs: 1733883078 + nsecs: 446942091 +frame_id: "camera_color_optical_frame" +[INFO] [1733883078.886147]: received mask_obs with header seq: 847 +stamp: + secs: 1733883078 + nsecs: 886965513 +frame_id: '' and grasped_object_mask header seq: 42219 +stamp: + secs: 1733883078 + nsecs: 646781921 +frame_id: "camera_color_optical_frame" +[INFO] [1733883079.091057]: received mask_obs with header seq: 848 +stamp: + secs: 1733883079 + nsecs: 91971158 +frame_id: '' and grasped_object_mask header seq: 42225 +stamp: + secs: 1733883078 + nsecs: 846621752 +frame_id: "camera_color_optical_frame" +[INFO] [1733883079.291188]: received mask_obs with header seq: 849 +stamp: + secs: 1733883079 + nsecs: 292060852 +frame_id: '' and grasped_object_mask header seq: 42231 +stamp: + secs: 1733883079 + nsecs: 46461821 +frame_id: "camera_color_optical_frame" +[INFO] [1733883079.483761]: received mask_obs with header seq: 850 +stamp: + secs: 1733883079 + nsecs: 484676361 +frame_id: '' and grasped_object_mask header seq: 42237 +stamp: + secs: 1733883079 + nsecs: 246300697 +frame_id: "camera_color_optical_frame" +[INFO] [1733883079.701560]: received mask_obs with header seq: 851 +stamp: + secs: 1733883079 + nsecs: 702465295 +frame_id: '' and grasped_object_mask header seq: 42243 +stamp: + secs: 1733883079 + nsecs: 446138382 +frame_id: "camera_color_optical_frame" +[INFO] [1733883079.887834]: received mask_obs with header seq: 852 +stamp: + secs: 1733883079 + nsecs: 888674497 +frame_id: '' and grasped_object_mask header seq: 42249 +stamp: + secs: 1733883079 + nsecs: 645975828 +frame_id: "camera_color_optical_frame" +[INFO] [1733883080.090843]: received mask_obs with header seq: 853 +stamp: + secs: 1733883080 + nsecs: 91479301 +frame_id: '' and grasped_object_mask header seq: 42255 +stamp: + secs: 1733883079 + nsecs: 845813274 +frame_id: "camera_color_optical_frame" +[INFO] [1733883080.292152]: received mask_obs with header seq: 854 +stamp: + secs: 1733883080 + nsecs: 293017148 +frame_id: '' and grasped_object_mask header seq: 42261 +stamp: + secs: 1733883080 + nsecs: 45650482 +frame_id: "camera_color_optical_frame" +[INFO] [1733883080.486146]: received mask_obs with header seq: 855 +stamp: + secs: 1733883080 + nsecs: 487071990 +frame_id: '' and grasped_object_mask header seq: 42267 +stamp: + secs: 1733883080 + nsecs: 245492220 +frame_id: "camera_color_optical_frame" +[INFO] [1733883080.684229]: received mask_obs with header seq: 856 +stamp: + secs: 1733883080 + nsecs: 684944152 +frame_id: '' and grasped_object_mask header seq: 42273 +stamp: + secs: 1733883080 + nsecs: 445335388 +frame_id: "camera_color_optical_frame" +[INFO] [1733883080.877372]: received mask_obs with header seq: 857 +stamp: + secs: 1733883080 + nsecs: 878116369 +frame_id: '' and grasped_object_mask header seq: 42279 +stamp: + secs: 1733883080 + nsecs: 645177841 +frame_id: "camera_color_optical_frame" +[INFO] [1733883081.089473]: received mask_obs with header seq: 858 +stamp: + secs: 1733883081 + nsecs: 90210914 +frame_id: '' and grasped_object_mask header seq: 42285 +stamp: + secs: 1733883080 + nsecs: 845020056 +frame_id: "camera_color_optical_frame" +[INFO] [1733883081.289819]: received mask_obs with header seq: 859 +stamp: + secs: 1733883081 + nsecs: 290548563 +frame_id: '' and grasped_object_mask header seq: 42291 +stamp: + secs: 1733883081 + nsecs: 44862509 +frame_id: "camera_color_optical_frame" +[INFO] [1733883081.476348]: received mask_obs with header seq: 860 +stamp: + secs: 1733883081 + nsecs: 477297067 +frame_id: '' and grasped_object_mask header seq: 42297 +stamp: + secs: 1733883081 + nsecs: 244699001 +frame_id: "camera_color_optical_frame" +[INFO] [1733883081.684732]: received mask_obs with header seq: 861 +stamp: + secs: 1733883081 + nsecs: 685504436 +frame_id: '' and grasped_object_mask header seq: 42303 +stamp: + secs: 1733883081 + nsecs: 444535017 +frame_id: "camera_color_optical_frame" +[INFO] [1733883081.884546]: received mask_obs with header seq: 862 +stamp: + secs: 1733883081 + nsecs: 885465383 +frame_id: '' and grasped_object_mask header seq: 42309 +stamp: + secs: 1733883081 + nsecs: 644371748 +frame_id: "camera_color_optical_frame" +[INFO] [1733883082.087006]: received mask_obs with header seq: 863 +stamp: + secs: 1733883082 + nsecs: 87851285 +frame_id: '' and grasped_object_mask header seq: 42315 +stamp: + secs: 1733883081 + nsecs: 844207287 +frame_id: "camera_color_optical_frame" +[INFO] [1733883082.282156]: received mask_obs with header seq: 864 +stamp: + secs: 1733883082 + nsecs: 283030509 +frame_id: '' and grasped_object_mask header seq: 42321 +stamp: + secs: 1733883082 + nsecs: 44042587 +frame_id: "camera_color_optical_frame" +[INFO] [1733883082.488995]: received mask_obs with header seq: 865 +stamp: + secs: 1733883082 + nsecs: 489531993 +frame_id: '' and grasped_object_mask header seq: 42327 +stamp: + secs: 1733883082 + nsecs: 243880987 +frame_id: "camera_color_optical_frame" +[INFO] [1733883082.693092]: received mask_obs with header seq: 866 +stamp: + secs: 1733883082 + nsecs: 693633317 +frame_id: '' and grasped_object_mask header seq: 42333 +stamp: + secs: 1733883082 + nsecs: 443720102 +frame_id: "camera_color_optical_frame" +[INFO] [1733883082.895522]: received mask_obs with header seq: 867 +stamp: + secs: 1733883082 + nsecs: 896270036 +frame_id: '' and grasped_object_mask header seq: 42339 +stamp: + secs: 1733883082 + nsecs: 643558979 +frame_id: "camera_color_optical_frame" +[INFO] [1733883083.104362]: received mask_obs with header seq: 868 +stamp: + secs: 1733883083 + nsecs: 103734970 +frame_id: '' and grasped_object_mask header seq: 42345 +stamp: + secs: 1733883082 + nsecs: 843398809 +frame_id: "camera_color_optical_frame" +[INFO] [1733883083.314759]: received mask_obs with header seq: 869 +stamp: + secs: 1733883083 + nsecs: 315489053 +frame_id: '' and grasped_object_mask header seq: 42351 +stamp: + secs: 1733883083 + nsecs: 43237686 +frame_id: "camera_color_optical_frame" +[INFO] [1733883083.521599]: received mask_obs with header seq: 870 +stamp: + secs: 1733883083 + nsecs: 522372007 +frame_id: '' and grasped_object_mask header seq: 42358 +stamp: + secs: 1733883083 + nsecs: 276379108 +frame_id: "camera_color_optical_frame" +[INFO] [1733883083.712913]: received mask_obs with header seq: 871 +stamp: + secs: 1733883083 + nsecs: 713778018 +frame_id: '' and grasped_object_mask header seq: 42364 +stamp: + secs: 1733883083 + nsecs: 476215363 +frame_id: "camera_color_optical_frame" +[INFO] [1733883083.919737]: received mask_obs with header seq: 872 +stamp: + secs: 1733883083 + nsecs: 920377016 +frame_id: '' and grasped_object_mask header seq: 42370 +stamp: + secs: 1733883083 + nsecs: 676050186 +frame_id: "camera_color_optical_frame" +[INFO] [1733883084.113025]: received mask_obs with header seq: 873 +stamp: + secs: 1733883084 + nsecs: 112495899 +frame_id: '' and grasped_object_mask header seq: 42376 +stamp: + secs: 1733883083 + nsecs: 875885248 +frame_id: "camera_color_optical_frame" +[INFO] [1733883084.320675]: received mask_obs with header seq: 874 +stamp: + secs: 1733883084 + nsecs: 321137428 +frame_id: '' and grasped_object_mask header seq: 42382 +stamp: + secs: 1733883084 + nsecs: 75720072 +frame_id: "camera_color_optical_frame" +[INFO] [1733883084.510699]: received mask_obs with header seq: 875 +stamp: + secs: 1733883084 + nsecs: 511530637 +frame_id: '' and grasped_object_mask header seq: 42388 +stamp: + secs: 1733883084 + nsecs: 275558710 +frame_id: "camera_color_optical_frame" +[INFO] [1733883084.715240]: received mask_obs with header seq: 876 +stamp: + secs: 1733883084 + nsecs: 715855598 +frame_id: '' and grasped_object_mask header seq: 42394 +stamp: + secs: 1733883084 + nsecs: 475396872 +frame_id: "camera_color_optical_frame" +[INFO] [1733883084.909891]: received mask_obs with header seq: 877 +stamp: + secs: 1733883084 + nsecs: 910677194 +frame_id: '' and grasped_object_mask header seq: 42400 +stamp: + secs: 1733883084 + nsecs: 675236464 +frame_id: "camera_color_optical_frame" +[INFO] [1733883085.101084]: received mask_obs with header seq: 878 +stamp: + secs: 1733883085 + nsecs: 101682424 +frame_id: '' and grasped_object_mask header seq: 42406 +stamp: + secs: 1733883084 + nsecs: 875074863 +frame_id: "camera_color_optical_frame" +[INFO] [1733883085.309487]: received mask_obs with header seq: 879 +stamp: + secs: 1733883085 + nsecs: 309807300 +frame_id: '' and grasped_object_mask header seq: 42412 +stamp: + secs: 1733883085 + nsecs: 74913979 +frame_id: "camera_color_optical_frame" +[INFO] [1733883085.501475]: received mask_obs with header seq: 880 +stamp: + secs: 1733883085 + nsecs: 502426862 +frame_id: '' and grasped_object_mask header seq: 42418 +stamp: + secs: 1733883085 + nsecs: 274759293 +frame_id: "camera_color_optical_frame" +[INFO] [1733883085.703914]: received mask_obs with header seq: 881 +stamp: + secs: 1733883085 + nsecs: 704722881 +frame_id: '' and grasped_object_mask header seq: 42424 +stamp: + secs: 1733883085 + nsecs: 474605083 +frame_id: "camera_color_optical_frame" +[INFO] [1733883085.904316]: received mask_obs with header seq: 882 +stamp: + secs: 1733883085 + nsecs: 904753923 +frame_id: '' and grasped_object_mask header seq: 42430 +stamp: + secs: 1733883085 + nsecs: 674450874 +frame_id: "camera_color_optical_frame" +[INFO] [1733883086.101123]: received mask_obs with header seq: 883 +stamp: + secs: 1733883086 + nsecs: 102050781 +frame_id: '' and grasped_object_mask header seq: 42436 +stamp: + secs: 1733883085 + nsecs: 874296904 +frame_id: "camera_color_optical_frame" +[INFO] [1733883086.295979]: received mask_obs with header seq: 884 +stamp: + secs: 1733883086 + nsecs: 296876192 +frame_id: '' and grasped_object_mask header seq: 42442 +stamp: + secs: 1733883086 + nsecs: 74143887 +frame_id: "camera_color_optical_frame" +[INFO] [1733883086.497347]: received mask_obs with header seq: 885 +stamp: + secs: 1733883086 + nsecs: 497634410 +frame_id: '' and grasped_object_mask header seq: 42447 +stamp: + secs: 1733883086 + nsecs: 240674019 +frame_id: "camera_color_optical_frame" +[INFO] [1733883086.699391]: received mask_obs with header seq: 886 +stamp: + secs: 1733883086 + nsecs: 700331449 +frame_id: '' and grasped_object_mask header seq: 42454 +stamp: + secs: 1733883086 + nsecs: 473817825 +frame_id: "camera_color_optical_frame" +[INFO] [1733883086.888006]: received mask_obs with header seq: 887 +stamp: + secs: 1733883086 + nsecs: 888107061 +frame_id: '' and grasped_object_mask header seq: 42459 +stamp: + secs: 1733883086 + nsecs: 640349150 +frame_id: "camera_color_optical_frame" +[INFO] [1733883087.081840]: received mask_obs with header seq: 888 +stamp: + secs: 1733883087 + nsecs: 82580804 +frame_id: '' and grasped_object_mask header seq: 42465 +stamp: + secs: 1733883086 + nsecs: 840185881 +frame_id: "camera_color_optical_frame" +[INFO] [1733883087.281461]: received mask_obs with header seq: 889 +stamp: + secs: 1733883087 + nsecs: 282369613 +frame_id: '' and grasped_object_mask header seq: 42471 +stamp: + secs: 1733883087 + nsecs: 40022135 +frame_id: "camera_color_optical_frame" +[INFO] [1733883087.482370]: received mask_obs with header seq: 890 +stamp: + secs: 1733883087 + nsecs: 483008861 +frame_id: '' and grasped_object_mask header seq: 42477 +stamp: + secs: 1733883087 + nsecs: 239856243 +frame_id: "camera_color_optical_frame" +[INFO] [1733883087.683968]: received mask_obs with header seq: 891 +stamp: + secs: 1733883087 + nsecs: 684839487 +frame_id: '' and grasped_object_mask header seq: 42483 +stamp: + secs: 1733883087 + nsecs: 439689398 +frame_id: "camera_color_optical_frame" +[INFO] [1733883087.892894]: received mask_obs with header seq: 892 +stamp: + secs: 1733883087 + nsecs: 892861843 +frame_id: '' and grasped_object_mask header seq: 42489 +stamp: + secs: 1733883087 + nsecs: 639522314 +frame_id: "camera_color_optical_frame" +[INFO] [1733883088.094871]: received mask_obs with header seq: 893 +stamp: + secs: 1733883088 + nsecs: 95611333 +frame_id: '' and grasped_object_mask header seq: 42495 +stamp: + secs: 1733883087 + nsecs: 839355707 +frame_id: "camera_color_optical_frame" +[INFO] [1733883088.281543]: received mask_obs with header seq: 894 +stamp: + secs: 1733883088 + nsecs: 282529115 +frame_id: '' and grasped_object_mask header seq: 42501 +stamp: + secs: 1733883088 + nsecs: 39187908 +frame_id: "camera_color_optical_frame" +[INFO] [1733883088.487901]: received mask_obs with header seq: 895 +stamp: + secs: 1733883088 + nsecs: 488756656 +frame_id: '' and grasped_object_mask header seq: 42508 +stamp: + secs: 1733883088 + nsecs: 272325993 +frame_id: "camera_color_optical_frame" +[INFO] [1733883088.692416]: received mask_obs with header seq: 896 +stamp: + secs: 1733883088 + nsecs: 692701101 +frame_id: '' and grasped_object_mask header seq: 42513 +stamp: + secs: 1733883088 + nsecs: 438853741 +frame_id: "camera_color_optical_frame" +[INFO] [1733883088.906863]: received mask_obs with header seq: 897 +stamp: + secs: 1733883088 + nsecs: 907102823 +frame_id: '' and grasped_object_mask header seq: 42519 +stamp: + secs: 1733883088 + nsecs: 638685942 +frame_id: "camera_color_optical_frame" +[INFO] [1733883089.108362]: received mask_obs with header seq: 898 +stamp: + secs: 1733883089 + nsecs: 109272241 +frame_id: '' and grasped_object_mask header seq: 42526 +stamp: + secs: 1733883088 + nsecs: 871823788 +frame_id: "camera_color_optical_frame" +[INFO] [1733883089.304040]: received mask_obs with header seq: 899 +stamp: + secs: 1733883089 + nsecs: 304779767 +frame_id: '' and grasped_object_mask header seq: 42532 +stamp: + secs: 1733883089 + nsecs: 71655273 +frame_id: "camera_color_optical_frame" +[INFO] [1733883089.517669]: received mask_obs with header seq: 900 +stamp: + secs: 1733883089 + nsecs: 518531799 +frame_id: '' and grasped_object_mask header seq: 42538 +stamp: + secs: 1733883089 + nsecs: 271488667 +frame_id: "camera_color_optical_frame" +[INFO] [1733883089.720506]: received mask_obs with header seq: 901 +stamp: + secs: 1733883089 + nsecs: 720213413 +frame_id: '' and grasped_object_mask header seq: 42545 +stamp: + secs: 1733883089 + nsecs: 504627943 +frame_id: "camera_color_optical_frame" +[INFO] [1733883089.926536]: received mask_obs with header seq: 902 +stamp: + secs: 1733883089 + nsecs: 927028656 +frame_id: '' and grasped_object_mask header seq: 42551 +stamp: + secs: 1733883089 + nsecs: 704460859 +frame_id: "camera_color_optical_frame" +[INFO] [1733883090.121912]: received mask_obs with header seq: 903 +stamp: + secs: 1733883090 + nsecs: 122742414 +frame_id: '' and grasped_object_mask header seq: 42556 +stamp: + secs: 1733883089 + nsecs: 870988846 +frame_id: "camera_color_optical_frame" +[INFO] [1733883090.336496]: received mask_obs with header seq: 904 +stamp: + secs: 1733883090 + nsecs: 337100505 +frame_id: '' and grasped_object_mask header seq: 42563 +stamp: + secs: 1733883090 + nsecs: 104127169 +frame_id: "camera_color_optical_frame" +[INFO] [1733883090.524952]: received mask_obs with header seq: 905 +stamp: + secs: 1733883090 + nsecs: 525888919 +frame_id: '' and grasped_object_mask header seq: 42569 +stamp: + secs: 1733883090 + nsecs: 303967476 +frame_id: "camera_color_optical_frame" +[INFO] [1733883090.735944]: received mask_obs with header seq: 906 +stamp: + secs: 1733883090 + nsecs: 736888885 +frame_id: '' and grasped_object_mask header seq: 42575 +stamp: + secs: 1733883090 + nsecs: 503807068 +frame_id: "camera_color_optical_frame" +[INFO] [1733883090.941684]: received mask_obs with header seq: 907 +stamp: + secs: 1733883090 + nsecs: 942602157 +frame_id: '' and grasped_object_mask header seq: 42581 +stamp: + secs: 1733883090 + nsecs: 703646660 +frame_id: "camera_color_optical_frame" +[INFO] [1733883091.144217]: received mask_obs with header seq: 908 +stamp: + secs: 1733883091 + nsecs: 144907951 +frame_id: '' and grasped_object_mask header seq: 42587 +stamp: + secs: 1733883090 + nsecs: 903486252 +frame_id: "camera_color_optical_frame" +[INFO] [1733883091.358659]: received mask_obs with header seq: 909 +stamp: + secs: 1733883091 + nsecs: 359580993 +frame_id: '' and grasped_object_mask header seq: 42593 +stamp: + secs: 1733883091 + nsecs: 103325367 +frame_id: "camera_color_optical_frame" +[INFO] [1733883091.564290]: received mask_obs with header seq: 910 +stamp: + secs: 1733883091 + nsecs: 564722061 +frame_id: '' and grasped_object_mask header seq: 42600 +stamp: + secs: 1733883091 + nsecs: 336468220 +frame_id: "camera_color_optical_frame" +[INFO] [1733883091.768316]: received mask_obs with header seq: 911 +stamp: + secs: 1733883091 + nsecs: 769130229 +frame_id: '' and grasped_object_mask header seq: 42606 +stamp: + secs: 1733883091 + nsecs: 536304474 +frame_id: "camera_color_optical_frame" +[INFO] [1733883091.967749]: received mask_obs with header seq: 912 +stamp: + secs: 1733883091 + nsecs: 968558073 +frame_id: '' and grasped_object_mask header seq: 42612 +stamp: + secs: 1733883091 + nsecs: 736140728 +frame_id: "camera_color_optical_frame" +[INFO] [1733883092.161025]: received mask_obs with header seq: 913 +stamp: + secs: 1733883092 + nsecs: 161980867 +frame_id: '' and grasped_object_mask header seq: 42618 +stamp: + secs: 1733883091 + nsecs: 935976744 +frame_id: "camera_color_optical_frame" +[INFO] [1733883092.365407]: received mask_obs with header seq: 914 +stamp: + secs: 1733883092 + nsecs: 366335630 +frame_id: '' and grasped_object_mask header seq: 42624 +stamp: + secs: 1733883092 + nsecs: 135814428 +frame_id: "camera_color_optical_frame" +[INFO] [1733883092.551846]: received mask_obs with header seq: 915 +stamp: + secs: 1733883092 + nsecs: 552690982 +frame_id: '' and grasped_object_mask header seq: 42630 +stamp: + secs: 1733883092 + nsecs: 335652351 +frame_id: "camera_color_optical_frame" +[INFO] [1733883092.759381]: received mask_obs with header seq: 916 +stamp: + secs: 1733883092 + nsecs: 760024547 +frame_id: '' and grasped_object_mask header seq: 42635 +stamp: + secs: 1733883092 + nsecs: 502183437 +frame_id: "camera_color_optical_frame" +[INFO] [1733883092.956099]: received mask_obs with header seq: 917 +stamp: + secs: 1733883092 + nsecs: 956745624 +frame_id: '' and grasped_object_mask header seq: 42642 +stamp: + secs: 1733883092 + nsecs: 735327959 +frame_id: "camera_color_optical_frame" +[INFO] [1733883093.157792]: received mask_obs with header seq: 918 +stamp: + secs: 1733883093 + nsecs: 158702611 +frame_id: '' and grasped_object_mask header seq: 42648 +stamp: + secs: 1733883092 + nsecs: 935165882 +frame_id: "camera_color_optical_frame" +[INFO] [1733883093.358092]: received mask_obs with header seq: 919 +stamp: + secs: 1733883093 + nsecs: 359010219 +frame_id: '' and grasped_object_mask header seq: 42654 +stamp: + secs: 1733883093 + nsecs: 135003567 +frame_id: "camera_color_optical_frame" +[INFO] [1733883093.541506]: received mask_obs with header seq: 920 +stamp: + secs: 1733883093 + nsecs: 542361259 +frame_id: '' and grasped_object_mask header seq: 42659 +stamp: + secs: 1733883093 + nsecs: 301535845 +frame_id: "camera_color_optical_frame" +[INFO] [1733883093.759010]: received mask_obs with header seq: 921 +stamp: + secs: 1733883093 + nsecs: 759990930 +frame_id: '' and grasped_object_mask header seq: 42665 +stamp: + secs: 1733883093 + nsecs: 501374245 +frame_id: "camera_color_optical_frame" +[INFO] [1733883093.958368]: received mask_obs with header seq: 922 +stamp: + secs: 1733883093 + nsecs: 959301710 +frame_id: '' and grasped_object_mask header seq: 42672 +stamp: + secs: 1733883093 + nsecs: 734519482 +frame_id: "camera_color_optical_frame" +[INFO] [1733883094.156615]: received mask_obs with header seq: 923 +stamp: + secs: 1733883094 + nsecs: 157175302 +frame_id: '' and grasped_object_mask header seq: 42678 +stamp: + secs: 1733883093 + nsecs: 934357882 +frame_id: "camera_color_optical_frame" +[INFO] [1733883094.335391]: received mask_obs with header seq: 924 +stamp: + secs: 1733883094 + nsecs: 336301565 +frame_id: '' and grasped_object_mask header seq: 42683 +stamp: + secs: 1733883094 + nsecs: 100890160 +frame_id: "camera_color_optical_frame" +[INFO] [1733883094.536997]: received mask_obs with header seq: 925 +stamp: + secs: 1733883094 + nsecs: 537668943 +frame_id: '' and grasped_object_mask header seq: 42689 +stamp: + secs: 1733883094 + nsecs: 300727367 +frame_id: "camera_color_optical_frame" +[INFO] [1733883094.734143]: received mask_obs with header seq: 926 +stamp: + secs: 1733883094 + nsecs: 735053300 +frame_id: '' and grasped_object_mask header seq: 42695 +stamp: + secs: 1733883094 + nsecs: 500564098 +frame_id: "camera_color_optical_frame" +[INFO] [1733883094.785264]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1733883094.925555]: received mask_obs with header seq: 927 +stamp: + secs: 1733883094 + nsecs: 925747871 +frame_id: '' and grasped_object_mask header seq: 42701 +stamp: + secs: 1733883094 + nsecs: 700401068 +frame_id: "camera_color_optical_frame" +[INFO] [1733883095.528883]: Stored episode 5. +[INFO] [1733883095.533236]: Saving video... +[INFO] [1733883096.100983]: Video saved! +[INFO] [1733883096.158591]: resetting environment +[INFO] [1733883096.181209]: cleared current plan +[INFO] [1733883096.185826]: moving to home +[WARN] [1733883096.112392]: Plan exhausted +[WARN] [1733883096.146625]: Plan exhausted +[INFO] [1733883098.996348]: reached home +[INFO] [1733883098.999561]: reset action history +[INFO] [1733883101.852389]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! \ No newline at end of file diff --git a/210732/wandb/run-20241210_210738-r4m9fzdi/files/requirements.txt b/210732/wandb/run-20241210_210738-r4m9fzdi/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..192179a195956e252bda33f2342b4970a9de79d9 --- /dev/null +++ b/210732/wandb/run-20241210_210738-r4m9fzdi/files/requirements.txt @@ -0,0 +1,339 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.2 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-metadata.json b/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ceb78d6e4e248f0802fca676489eb942c6e4fc27 --- /dev/null +++ b/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2024-12-11T02:07:39.002284", + "startedAt": "2024-12-11T02:07:38.614206", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "4f7c0421eb84a7caf6bf5670f68777523362676f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 4312.5005, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4499.997, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.998, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.956, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 367.3990364074707 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71596908569336 + } +} diff --git a/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json b/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..23fc7a532eaaed2a610c84ae966daccc08efd124 --- /dev/null +++ b/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "daf03ac4df433b6afa8025ee9f7d8ed0bed270796e3304bfc12b399ae0335495", "size": 1152674, "path": "media/videos/eval/0_eval_0_daf03ac4df433b6afa80.mp4"}, "global_step": 678, "_timestamp": 1733883104.2055726, "_runtime": 245.5830535888672, "_step": 9, "eval/1_eval": {"_type": "video-file", "sha256": "fa72f11620e871983c64a5c2dc859cc7580356d19b495eb0134d04e0f3ef9263", "size": 1175556, "path": "media/videos/eval/1_eval_1_fa72f11620e871983c64.mp4"}, "eval/num_success": 5.0, "episode": 5.0, "eval/success_rate": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "b17305aedebc761a4898e8d4ff0fd8cd6ab34922d528b2e1833fe396b670cfbe", "size": 1170247, "path": "media/videos/eval/2_eval_2_b17305aedebc761a4898.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "49c63cb7694dee4c930a2f65bf4b4cf1d5025f26cca5462b29f9cd8b7b7aa953", "size": 1171672, "path": "media/videos/eval/3_eval_3_49c63cb7694dee4c930a.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "a244b0b09dc15d2eae1cd3aaa59edb276ea040384e1f9010b1b996ca7083d6e5", "size": 1157328, "path": "media/videos/eval/4_eval_4_a244b0b09dc15d2eae1c.mp4"}, "_wandb": {"runtime": 245}} \ No newline at end of file diff --git a/210732/wandb/run-20241210_210738-r4m9fzdi/logs/debug-internal.log b/210732/wandb/run-20241210_210738-r4m9fzdi/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ba4ae0369f190c7e02df864203e4dfd8110c033c --- /dev/null +++ b/210732/wandb/run-20241210_210738-r4m9fzdi/logs/debug-internal.log @@ -0,0 +1,742 @@ +2024-12-10 21:07:38,622 INFO StreamThr :601021 [internal.py:wandb_internal():85] W&B internal server running at pid: 601021, started at: 2024-12-10 21:07:38.622053 +2024-12-10 21:07:38,623 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status +2024-12-10 21:07:38,634 INFO WriterThread:601021 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/run-r4m9fzdi.wandb +2024-12-10 21:07:38,634 DEBUG SenderThread:601021 [sender.py:send():379] send: header +2024-12-10 21:07:38,645 DEBUG SenderThread:601021 [sender.py:send():379] send: run +2024-12-10 21:07:38,918 INFO SenderThread:601021 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files +2024-12-10 21:07:38,918 INFO SenderThread:601021 [sender.py:_start_run_threads():1188] run started: r4m9fzdi with start time 1733882858.622519 +2024-12-10 21:07:38,936 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: check_version +2024-12-10 21:07:38,936 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: check_version +2024-12-10 21:07:38,984 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: run_start +2024-12-10 21:07:38,989 DEBUG HandlerThread:601021 [system_info.py:__init__():26] System info init +2024-12-10 21:07:38,989 DEBUG HandlerThread:601021 [system_info.py:__init__():41] System info init done +2024-12-10 21:07:38,989 INFO HandlerThread:601021 [system_monitor.py:start():194] Starting system monitor +2024-12-10 21:07:38,989 INFO SystemMonitor:601021 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-10 21:07:38,989 INFO HandlerThread:601021 [system_monitor.py:probe():214] Collecting system info +2024-12-10 21:07:38,989 INFO SystemMonitor:601021 [interfaces.py:start():188] Started cpu monitoring +2024-12-10 21:07:38,990 INFO SystemMonitor:601021 [interfaces.py:start():188] Started disk monitoring +2024-12-10 21:07:38,991 INFO SystemMonitor:601021 [interfaces.py:start():188] Started gpu monitoring +2024-12-10 21:07:38,991 INFO SystemMonitor:601021 [interfaces.py:start():188] Started memory monitoring +2024-12-10 21:07:38,992 INFO SystemMonitor:601021 [interfaces.py:start():188] Started network monitoring +2024-12-10 21:07:39,002 DEBUG HandlerThread:601021 [system_info.py:probe():152] Probing system +2024-12-10 21:07:39,003 DEBUG HandlerThread:601021 [system_info.py:_probe_git():137] Probing git +2024-12-10 21:07:39,008 DEBUG HandlerThread:601021 [system_info.py:_probe_git():145] Probing git done +2024-12-10 21:07:39,008 DEBUG HandlerThread:601021 [system_info.py:probe():200] Probing system done +2024-12-10 21:07:39,008 DEBUG HandlerThread:601021 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-11T02:07:39.002284', 'startedAt': '2024-12-11T02:07:38.614206', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '4f7c0421eb84a7caf6bf5670f68777523362676f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4312.5005, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.997, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.956, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 367.3990364074707}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71596908569336}} +2024-12-10 21:07:39,008 INFO HandlerThread:601021 [system_monitor.py:probe():224] Finished collecting system info +2024-12-10 21:07:39,008 INFO HandlerThread:601021 [system_monitor.py:probe():227] Publishing system info +2024-12-10 21:07:39,008 DEBUG HandlerThread:601021 [system_info.py:_save_code():44] Saving code +2024-12-10 21:07:39,010 DEBUG HandlerThread:601021 [system_info.py:_save_code():67] Saving code done +2024-12-10 21:07:39,010 DEBUG HandlerThread:601021 [system_info.py:_save_patches():84] Saving git patches +2024-12-10 21:07:39,074 DEBUG HandlerThread:601021 [system_info.py:_save_patches():126] Saving git patches done +2024-12-10 21:07:39,074 INFO HandlerThread:601021 [system_monitor.py:probe():229] Finished publishing system info +2024-12-10 21:07:39,077 DEBUG SenderThread:601021 [sender.py:send():379] send: files +2024-12-10 21:07:39,077 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-10 21:07:39,077 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-10 21:07:39,078 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-10 21:07:39,335 INFO wandb-upload_0:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/19y6xz09-wandb-metadata.json +2024-12-10 21:07:39,342 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: python_packages +2024-12-10 21:07:39,342 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: python_packages +2024-12-10 21:07:39,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:07:39,345 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:07:39,351 INFO wandb-upload_2:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/yyukg9ez-diff.patch +2024-12-10 21:07:39,386 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:39,386 INFO HandlerThread:601021 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/tb" + save: true +} +control { + relay_id: "0d9e67a43a6245129e461b3eb5978722" +} +_info { + stream_id: "r4m9fzdi" +} + +2024-12-10 21:07:39,386 DEBUG HandlerThread:601021 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-10 21:07:39,459 INFO wandb-upload_1:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/6dtumhkg-code/FISH/eval_robot.py +2024-12-10 21:07:39,463 DEBUG SenderThread:601021 [sender.py:send():379] send: telemetry +2024-12-10 21:07:39,463 DEBUG SenderThread:601021 [sender.py:send():379] send: tbrecord +2024-12-10 21:07:39,464 DEBUG SenderThread:601021 [sender.py:send():379] send: files +2024-12-10 21:07:39,464 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 with policy live +2024-12-10 21:07:39,666 INFO wandb-upload_0:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/0lfrfmoc-events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 +2024-12-10 21:07:39,920 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/code/FISH/eval_robot.py +2024-12-10 21:07:39,920 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/diff.patch +2024-12-10 21:07:39,920 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 +2024-12-10 21:07:39,920 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:07:39,920 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/requirements.txt +2024-12-10 21:07:39,920 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-metadata.json +2024-12-10 21:07:39,920 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/code +2024-12-10 21:07:39,920 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/code/FISH +2024-12-10 21:07:40,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:41,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:41,919 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:07:42,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:42,752 DEBUG SenderThread:601021 [sender.py:send():379] send: config +2024-12-10 21:07:43,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:43,760 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:07:43,920 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:07:44,345 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:45,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:45,920 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:07:46,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:47,345 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:47,921 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:07:48,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:49,343 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:07:49,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:49,921 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:07:50,345 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:51,345 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:51,922 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:07:52,345 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:53,346 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:54,342 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:07:54,342 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:07:54,385 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:54,435 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:07:55,346 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:56,346 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:57,346 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:58,346 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:07:59,346 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:00,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:00,346 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:01,346 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:02,346 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:03,347 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:04,347 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:05,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:05,347 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:06,347 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:07,347 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:07,926 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:08,347 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:09,342 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:08:09,343 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:08:09,385 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:10,348 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:10,376 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:10,927 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/config.yaml +2024-12-10 21:08:11,348 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:11,927 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:12,348 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:13,348 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:13,928 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:14,348 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:15,348 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:15,562 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:15,928 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:16,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:17,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:17,929 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:18,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:19,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:19,930 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:20,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:20,631 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:21,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:21,930 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:22,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:23,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:23,931 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:24,342 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:08:24,343 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:08:24,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:25,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:25,779 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:25,931 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:26,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:27,349 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:27,933 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:28,350 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:29,350 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:29,933 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:30,350 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:30,977 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:31,350 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:31,934 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:32,350 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:33,350 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:33,934 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:34,350 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:35,350 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:35,935 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:36,166 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:36,350 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:37,350 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:37,935 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:38,229 DEBUG SenderThread:601021 [sender.py:send():379] send: files +2024-12-10 21:08:38,229 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_daf03ac4df433b6afa80.mp4 with policy now +2024-12-10 21:08:38,229 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:08:38,230 DEBUG SenderThread:601021 [sender.py:send():379] send: history +2024-12-10 21:08:38,230 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:08:38,231 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:08:38,351 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:38,482 INFO wandb-upload_1:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/pmuvl3df-media/videos/eval/0_eval_0_daf03ac4df433b6afa80.mp4 +2024-12-10 21:08:38,936 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json +2024-12-10 21:08:38,936 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/0_eval_0_daf03ac4df433b6afa80.mp4 +2024-12-10 21:08:38,936 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval +2024-12-10 21:08:38,936 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos +2024-12-10 21:08:38,936 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media +2024-12-10 21:08:38,992 DEBUG SystemMonitor:601021 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-10 21:08:38,992 DEBUG SenderThread:601021 [sender.py:send():379] send: stats +2024-12-10 21:08:39,235 INFO wandb-upload_0:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/40lm6lse-media/videos/eval/0_eval_0_daf03ac4df433b6afa80.mp4 +2024-12-10 21:08:39,343 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:08:39,343 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:08:39,389 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:39,936 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:40,351 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:41,351 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:41,937 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:42,086 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:42,352 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:43,351 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:44,351 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:45,352 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:45,938 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:08:46,352 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:47,345 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:47,352 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:48,352 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:49,353 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:50,353 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:51,353 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:52,346 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:52,353 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:53,353 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:54,343 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:08:54,343 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:08:54,385 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:55,354 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:56,354 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:57,354 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:57,445 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:08:58,354 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:08:59,354 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:00,354 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:01,354 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:01,943 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 +2024-12-10 21:09:02,197 INFO wandb-upload_1:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/2q7yeh2p-events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 +2024-12-10 21:09:02,354 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:03,246 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:09:03,354 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:03,944 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:04,355 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:05,355 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:05,945 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:06,355 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:07,355 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:07,945 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:08,355 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:08,417 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:09:08,993 DEBUG SenderThread:601021 [sender.py:send():379] send: stats +2024-12-10 21:09:09,343 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:09:09,343 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:09:09,356 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:09,946 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:10,355 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:11,355 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:11,947 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:12,356 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:13,356 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:13,420 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:09:13,947 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:14,356 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:15,356 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:15,948 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:16,356 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:17,357 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:17,949 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:18,357 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:18,442 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:09:19,358 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:19,950 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:20,360 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:21,359 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:21,950 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:22,359 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:23,360 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:23,619 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:09:23,951 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:24,343 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:09:24,343 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:09:24,360 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:25,360 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:25,952 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:26,360 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:27,360 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:27,952 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:28,360 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:28,639 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:09:29,360 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:29,953 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:30,360 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:31,360 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:31,954 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:32,361 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:33,366 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:33,618 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:09:33,618 DEBUG SenderThread:601021 [sender.py:send():379] send: files +2024-12-10 21:09:33,619 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_fa72f11620e871983c64.mp4 with policy now +2024-12-10 21:09:33,619 DEBUG SenderThread:601021 [sender.py:send():379] send: history +2024-12-10 21:09:33,620 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:09:33,620 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:09:33,879 INFO wandb-upload_0:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/rlxqu1db-media/videos/eval/1_eval_1_fa72f11620e871983c64.mp4 +2024-12-10 21:09:33,955 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:33,955 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json +2024-12-10 21:09:33,955 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/1_eval_1_fa72f11620e871983c64.mp4 +2024-12-10 21:09:33,955 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval +2024-12-10 21:09:34,255 INFO wandb-upload_1:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/69opme6e-media/videos/eval/1_eval_1_fa72f11620e871983c64.mp4 +2024-12-10 21:09:34,351 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:09:34,363 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:35,371 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:35,955 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:36,364 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:37,367 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:37,956 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:38,369 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:38,993 DEBUG SenderThread:601021 [sender.py:send():379] send: stats +2024-12-10 21:09:39,343 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:09:39,343 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:09:39,364 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:39,480 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:09:39,957 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:40,364 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:41,365 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:41,909 DEBUG SenderThread:601021 [sender.py:send():379] send: history +2024-12-10 21:09:41,909 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:09:41,910 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:09:41,957 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json +2024-12-10 21:09:41,958 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 +2024-12-10 21:09:42,366 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:43,365 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:43,958 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:44,365 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:45,182 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:09:45,366 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:45,959 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:46,365 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:47,365 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:47,960 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:48,365 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:49,368 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:49,960 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:50,248 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:09:50,368 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:51,368 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:51,961 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:52,368 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:53,369 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:53,962 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:54,343 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:09:54,343 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:09:54,369 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:55,369 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:55,455 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:09:55,963 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:56,369 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:57,369 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:57,964 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:09:58,369 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:59,369 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:09:59,964 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:00,369 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:00,638 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:01,370 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:01,965 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:02,370 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:03,370 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:03,966 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:04,370 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:05,370 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:05,671 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:05,967 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:06,370 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:07,371 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:07,967 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:08,371 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:08,994 DEBUG SenderThread:601021 [sender.py:send():379] send: stats +2024-12-10 21:10:09,343 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:10:09,344 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:10:09,371 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:09,968 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:10,371 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:10,843 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:11,371 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:11,969 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:12,371 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:13,373 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:13,970 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:14,025 DEBUG SenderThread:601021 [sender.py:send():379] send: files +2024-12-10 21:10:14,025 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_b17305aedebc761a4898.mp4 with policy now +2024-12-10 21:10:14,026 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:10:14,027 DEBUG SenderThread:601021 [sender.py:send():379] send: history +2024-12-10 21:10:14,027 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:10:14,028 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:10:14,282 INFO wandb-upload_0:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/a215d27a-media/videos/eval/2_eval_2_b17305aedebc761a4898.mp4 +2024-12-10 21:10:14,374 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:14,970 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json +2024-12-10 21:10:14,970 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/2_eval_2_b17305aedebc761a4898.mp4 +2024-12-10 21:10:14,970 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval +2024-12-10 21:10:15,275 INFO wandb-upload_1:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/zmle0ig3-media/videos/eval/2_eval_2_b17305aedebc761a4898.mp4 +2024-12-10 21:10:15,388 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:15,970 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:16,353 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:16,376 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:17,378 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:17,971 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:18,389 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:19,379 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:20,380 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:21,387 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:21,690 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:21,972 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:22,387 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:22,672 DEBUG SenderThread:601021 [sender.py:send():379] send: history +2024-12-10 21:10:22,672 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:10:22,695 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:10:22,973 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json +2024-12-10 21:10:22,973 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 +2024-12-10 21:10:23,233 INFO wandb-upload_0:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/qp1en2ki-events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 +2024-12-10 21:10:23,384 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:23,973 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:24,347 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:10:24,347 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:10:24,389 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:25,393 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:25,974 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:26,386 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:26,987 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:27,387 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:27,975 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:28,386 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:29,387 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:29,975 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:30,387 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:31,387 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:31,976 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:32,011 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:32,387 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:33,387 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:33,977 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:34,387 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:35,387 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:35,978 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:36,387 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:37,171 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:37,388 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:37,978 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:38,388 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:38,994 DEBUG SenderThread:601021 [sender.py:send():379] send: stats +2024-12-10 21:10:39,344 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:10:39,344 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:10:39,388 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:39,979 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:40,388 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:41,388 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:41,980 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:42,176 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:42,388 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:43,388 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:43,981 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:44,388 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:45,388 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:45,982 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:46,389 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:47,263 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:47,389 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:47,982 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:48,389 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:49,389 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:49,983 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:50,389 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:51,390 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:51,984 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:52,390 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:52,404 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:53,390 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:53,985 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:54,352 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:10:54,352 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:10:54,394 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:55,036 DEBUG SenderThread:601021 [sender.py:send():379] send: files +2024-12-10 21:10:55,036 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_49c63cb7694dee4c930a.mp4 with policy now +2024-12-10 21:10:55,043 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:10:55,044 DEBUG SenderThread:601021 [sender.py:send():379] send: history +2024-12-10 21:10:55,044 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:10:55,044 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:10:55,288 INFO wandb-upload_1:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/d9c1a6dv-media/videos/eval/3_eval_3_49c63cb7694dee4c930a.mp4 +2024-12-10 21:10:55,393 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:55,985 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:10:55,986 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json +2024-12-10 21:10:55,986 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/3_eval_3_49c63cb7694dee4c930a.mp4 +2024-12-10 21:10:55,986 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval +2024-12-10 21:10:56,283 INFO wandb-upload_0:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/rmi7s53d-media/videos/eval/3_eval_3_49c63cb7694dee4c930a.mp4 +2024-12-10 21:10:56,399 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:57,401 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:57,872 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:10:58,405 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:59,406 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:10:59,987 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:00,410 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:01,409 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:01,987 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:02,413 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:03,405 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:11:03,449 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:03,718 DEBUG SenderThread:601021 [sender.py:send():379] send: history +2024-12-10 21:11:03,718 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:11:03,718 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:11:03,988 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:03,988 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json +2024-12-10 21:11:03,988 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 +2024-12-10 21:11:04,415 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:05,414 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:05,989 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:06,419 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:07,416 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:07,990 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:08,415 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:08,464 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:11:08,995 DEBUG SenderThread:601021 [sender.py:send():379] send: stats +2024-12-10 21:11:09,345 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:11:09,345 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:11:09,415 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:09,990 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:10,416 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:11,415 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:11,991 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:12,416 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:13,416 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:13,485 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:11:13,992 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:14,416 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:15,416 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:15,993 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:16,416 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:17,416 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:17,993 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:18,417 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:18,684 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:11:19,417 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:19,994 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:20,417 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:21,417 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:21,995 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:22,417 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:23,417 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:23,723 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:11:23,996 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:24,345 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:11:24,345 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:11:24,417 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:25,417 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:25,996 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:26,417 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:27,417 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:27,997 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:28,418 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:28,907 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:11:29,418 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:29,998 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:30,418 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:31,418 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:32,418 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:33,001 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:33,420 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:33,958 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:11:34,419 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:35,002 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:35,424 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:36,155 DEBUG SenderThread:601021 [sender.py:send():379] send: files +2024-12-10 21:11:36,155 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_a244b0b09dc15d2eae1c.mp4 with policy now +2024-12-10 21:11:36,158 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: partial_history +2024-12-10 21:11:36,158 DEBUG SenderThread:601021 [sender.py:send():379] send: history +2024-12-10 21:11:36,158 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:11:36,158 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:11:36,398 INFO wandb-upload_1:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/knvvnylb-media/videos/eval/4_eval_4_a244b0b09dc15d2eae1c.mp4 +2024-12-10 21:11:36,426 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:37,003 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:37,003 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json +2024-12-10 21:11:37,003 INFO Thread-12 :601021 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/4_eval_4_a244b0b09dc15d2eae1c.mp4 +2024-12-10 21:11:37,003 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval +2024-12-10 21:11:37,309 INFO wandb-upload_0:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/c2ykdz2u-media/videos/eval/4_eval_4_a244b0b09dc15d2eae1c.mp4 +2024-12-10 21:11:37,424 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:38,424 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:38,995 DEBUG SenderThread:601021 [sender.py:send():379] send: stats +2024-12-10 21:11:38,996 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:11:39,003 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:39,357 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: stop_status +2024-12-10 21:11:39,357 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: stop_status +2024-12-10 21:11:39,427 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:40,449 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:41,004 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:41,445 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:42,447 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:43,446 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:44,205 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:11:44,250 DEBUG SenderThread:601021 [sender.py:send():379] send: exit +2024-12-10 21:11:44,250 INFO SenderThread:601021 [sender.py:send_exit():586] handling exit code: 0 +2024-12-10 21:11:44,250 INFO SenderThread:601021 [sender.py:send_exit():588] handling runtime: 245 +2024-12-10 21:11:44,250 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:11:44,251 INFO SenderThread:601021 [sender.py:send_exit():594] send defer +2024-12-10 21:11:44,251 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:44,251 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-10 21:11:44,251 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:44,251 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-10 21:11:44,251 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 1 +2024-12-10 21:11:44,251 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:44,251 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-10 21:11:44,251 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:44,251 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-10 21:11:44,251 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 2 +2024-12-10 21:11:44,251 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:44,251 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-10 21:11:44,251 INFO HandlerThread:601021 [system_monitor.py:finish():203] Stopping system monitor +2024-12-10 21:11:44,251 DEBUG SystemMonitor:601021 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-10 21:11:44,252 INFO HandlerThread:601021 [interfaces.py:finish():200] Joined cpu monitor +2024-12-10 21:11:44,252 DEBUG SystemMonitor:601021 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-10 21:11:44,252 INFO HandlerThread:601021 [interfaces.py:finish():200] Joined disk monitor +2024-12-10 21:11:44,304 INFO HandlerThread:601021 [interfaces.py:finish():200] Joined gpu monitor +2024-12-10 21:11:44,304 INFO HandlerThread:601021 [interfaces.py:finish():200] Joined memory monitor +2024-12-10 21:11:44,304 INFO HandlerThread:601021 [interfaces.py:finish():200] Joined network monitor +2024-12-10 21:11:44,304 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:44,305 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-10 21:11:44,305 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 3 +2024-12-10 21:11:44,305 DEBUG SenderThread:601021 [sender.py:send():379] send: stats +2024-12-10 21:11:44,305 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:44,305 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-10 21:11:44,305 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:44,305 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-10 21:11:44,305 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 4 +2024-12-10 21:11:44,305 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:44,305 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-10 21:11:45,006 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:45,006 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json +2024-12-10 21:11:45,006 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 +2024-12-10 21:11:45,246 INFO wandb-upload_1:601021 [upload_job.py:push():130] Uploaded file /tmp/tmp8arqdxjawandb/r96vgupj-events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 +2024-12-10 21:11:47,006 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:50,307 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:11:50,308 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:11:50,308 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:50,308 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: keepalive +2024-12-10 21:11:50,308 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-10 21:11:50,308 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 5 +2024-12-10 21:11:50,308 DEBUG SenderThread:601021 [sender.py:send():379] send: history +2024-12-10 21:11:50,308 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:50,308 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:11:50,308 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-10 21:11:50,309 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:11:50,309 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:11:50,309 DEBUG SenderThread:601021 [sender.py:send():379] send: history +2024-12-10 21:11:50,309 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: summary_record +2024-12-10 21:11:50,313 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:11:50,313 DEBUG SenderThread:601021 [sender.py:send():379] send: summary +2024-12-10 21:11:50,313 INFO SenderThread:601021 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-10 21:11:50,313 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:50,314 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-10 21:11:50,314 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 6 +2024-12-10 21:11:50,314 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:50,314 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-10 21:11:50,314 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:50,314 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-10 21:11:50,314 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 7 +2024-12-10 21:11:50,314 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: status_report +2024-12-10 21:11:50,314 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:50,314 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-10 21:11:50,314 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:50,314 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-10 21:11:51,008 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json +2024-12-10 21:11:51,251 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:11:52,132 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 8 +2024-12-10 21:11:52,132 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:11:52,132 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:52,133 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-10 21:11:52,133 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:52,133 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-10 21:11:52,133 INFO SenderThread:601021 [job_builder.py:build():440] Attempting to build job artifact +2024-12-10 21:11:52,136 INFO SenderThread:601021 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-10 21:11:52,151 INFO SenderThread:601021 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-10 21:11:52,160 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 9 +2024-12-10 21:11:52,160 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:52,160 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-10 21:11:52,160 DEBUG SenderThread:601021 [sender.py:send():379] send: artifact +2024-12-10 21:11:52,252 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:11:52,718 INFO wandb-upload_2:601021 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmphlyt7eoj +2024-12-10 21:11:52,983 INFO wandb-upload_1:601021 [upload_job.py:push():88] Uploaded file /tmp/tmpsi_q7yxg/wandb-job.json +2024-12-10 21:11:53,006 INFO wandb-upload_0:601021 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpk9mk35pr +2024-12-10 21:11:53,009 INFO Thread-12 :601021 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:53,692 INFO SenderThread:601021 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM2Njc3OTY4NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM2Njc3NTA2NA==', 'versionIndex': 338}}} +2024-12-10 21:11:53,692 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:53,692 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-10 21:11:53,692 INFO SenderThread:601021 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-10 21:11:54,009 INFO SenderThread:601021 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files +2024-12-10 21:11:54,010 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/config.yaml config.yaml +2024-12-10 21:11:54,010 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json wandb-summary.json +2024-12-10 21:11:54,012 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/requirements.txt requirements.txt +2024-12-10 21:11:54,012 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-metadata.json wandb-metadata.json +2024-12-10 21:11:54,013 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 events.out.tfevents.1733882859.leonmkim-ROG-Strix-G15CS-G15CS.600897.0 +2024-12-10 21:11:54,015 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log output.log +2024-12-10 21:11:54,015 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/diff.patch diff.patch +2024-12-10 21:11:54,016 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-10 21:11:54,017 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/0_eval_0_daf03ac4df433b6afa80.mp4 media/videos/eval/0_eval_0_daf03ac4df433b6afa80.mp4 +2024-12-10 21:11:54,017 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/4_eval_4_a244b0b09dc15d2eae1c.mp4 media/videos/eval/4_eval_4_a244b0b09dc15d2eae1c.mp4 +2024-12-10 21:11:54,017 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/3_eval_3_49c63cb7694dee4c930a.mp4 media/videos/eval/3_eval_3_49c63cb7694dee4c930a.mp4 +2024-12-10 21:11:54,017 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/2_eval_2_b17305aedebc761a4898.mp4 media/videos/eval/2_eval_2_b17305aedebc761a4898.mp4 +2024-12-10 21:11:54,017 INFO SenderThread:601021 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/media/videos/eval/1_eval_1_fa72f11620e871983c64.mp4 media/videos/eval/1_eval_1_fa72f11620e871983c64.mp4 +2024-12-10 21:11:54,017 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 10 +2024-12-10 21:11:54,017 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:11:54,017 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:54,017 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-10 21:11:54,018 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:54,018 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-10 21:11:54,018 INFO SenderThread:601021 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 21:11:54,206 INFO wandb-upload_0:601021 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/wandb-summary.json +2024-12-10 21:11:54,206 INFO wandb-upload_1:601021 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/requirements.txt +2024-12-10 21:11:54,248 INFO wandb-upload_2:601021 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/config.yaml +2024-12-10 21:11:54,252 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:11:54,253 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:11:54,282 INFO wandb-upload_3:601021 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/files/output.log +2024-12-10 21:11:54,482 INFO Thread-11 (_thread_body):601021 [sender.py:transition_state():614] send defer: 11 +2024-12-10 21:11:54,482 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:54,482 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-10 21:11:54,483 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:54,483 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-10 21:11:54,483 INFO SenderThread:601021 [file_pusher.py:join():175] waiting for file pusher +2024-12-10 21:11:54,484 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 12 +2024-12-10 21:11:54,484 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:54,484 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-10 21:11:54,484 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:54,484 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-10 21:11:54,484 INFO SenderThread:601021 [file_stream.py:finish():601] file stream finish called +2024-12-10 21:11:54,670 INFO SenderThread:601021 [file_stream.py:finish():605] file stream finish is done +2024-12-10 21:11:54,670 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 13 +2024-12-10 21:11:54,670 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:54,671 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-10 21:11:54,671 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:54,671 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-10 21:11:54,671 INFO SenderThread:601021 [sender.py:transition_state():614] send defer: 14 +2024-12-10 21:11:54,671 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: defer +2024-12-10 21:11:54,671 INFO HandlerThread:601021 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-10 21:11:54,671 DEBUG SenderThread:601021 [sender.py:send():379] send: final +2024-12-10 21:11:54,672 DEBUG SenderThread:601021 [sender.py:send():379] send: footer +2024-12-10 21:11:54,672 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: defer +2024-12-10 21:11:54,672 INFO SenderThread:601021 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-10 21:11:54,672 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:11:54,673 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:11:54,673 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: server_info +2024-12-10 21:11:54,673 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: server_info +2024-12-10 21:11:54,675 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-10 21:11:54,675 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: get_summary +2024-12-10 21:11:54,676 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-10 21:11:54,676 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-10 21:11:54,767 DEBUG SenderThread:601021 [sender.py:send_request():406] send_request: poll_exit +2024-12-10 21:11:54,768 INFO MainThread:601021 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-10 21:11:54,768 INFO MainThread:601021 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-10 21:11:54,768 INFO MainThread:601021 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-10 21:11:54,768 DEBUG HandlerThread:601021 [handler.py:handle_request():158] handle_request: shutdown +2024-12-10 21:11:54,768 INFO HandlerThread:601021 [handler.py:finish():882] shutting down handler +2024-12-10 21:11:55,675 INFO WriterThread:601021 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/run-r4m9fzdi.wandb +2024-12-10 21:11:55,768 INFO SenderThread:601021 [sender.py:finish():1615] shutting down sender +2024-12-10 21:11:55,768 INFO SenderThread:601021 [file_pusher.py:finish():169] shutting down file pusher +2024-12-10 21:11:55,768 INFO SenderThread:601021 [file_pusher.py:join():175] waiting for file pusher diff --git a/210732/wandb/run-20241210_210738-r4m9fzdi/logs/debug.log b/210732/wandb/run-20241210_210738-r4m9fzdi/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9d046d8e75bfaab93f97ddbf392b64c702202e21 --- /dev/null +++ b/210732/wandb/run-20241210_210738-r4m9fzdi/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_setup.py:_flush():76] Configure stats pid to 600897 +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/settings +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/logs/debug.log +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/wandb/run-20241210_210738-r4m9fzdi/logs/debug-internal.log +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_init.py:init():569] calling init triggers +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '265626', 'wandb_notes': '265626_req_265827', 'eval': True, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim_left_to_right'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'obs_shape': [5, 240, 320], 'action_shape': [36, 7], 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}, 'policy_frequency': 5, 'interpolation_frequency': 25}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'true_action_history': False, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt'} +2024-12-10 21:07:38,617 INFO MainThread:600897 [wandb_init.py:init():619] starting backend +2024-12-10 21:07:38,618 INFO MainThread:600897 [wandb_init.py:init():623] setting up manager +2024-12-10 21:07:38,621 INFO MainThread:600897 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-10 21:07:38,622 INFO MainThread:600897 [wandb_init.py:init():631] backend started and connected +2024-12-10 21:07:38,633 INFO MainThread:600897 [wandb_init.py:init():720] updated telemetry +2024-12-10 21:07:38,642 INFO MainThread:600897 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-10 21:07:38,935 INFO MainThread:600897 [wandb_run.py:_on_init():2435] communicating current version +2024-12-10 21:07:38,977 INFO MainThread:600897 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.0 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-10 21:07:38,978 INFO MainThread:600897 [wandb_init.py:init():804] starting run threads in backend +2024-12-10 21:07:39,343 INFO MainThread:600897 [wandb_run.py:_console_start():2413] atexit reg +2024-12-10 21:07:39,343 INFO MainThread:600897 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-10 21:07:39,343 INFO MainThread:600897 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-10 21:07:39,344 INFO MainThread:600897 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-10 21:07:39,347 INFO MainThread:600897 [wandb_init.py:init():847] run started, returning control to user process +2024-12-10 21:07:39,347 INFO MainThread:600897 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265626/210732/tb, True +2024-12-10 21:07:42,752 INFO MainThread:600897 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-10 21:11:55,771 WARNING MsgRouterThr:600897 [router.py:message_loop():77] message_loop has been closed diff --git a/210732/wandb/run-20241210_210738-r4m9fzdi/run-r4m9fzdi.wandb b/210732/wandb/run-20241210_210738-r4m9fzdi/run-r4m9fzdi.wandb new file mode 100644 index 0000000000000000000000000000000000000000..21adcfa272d910a7a95eba0cd52c33e1a372fe2c --- /dev/null +++ b/210732/wandb/run-20241210_210738-r4m9fzdi/run-r4m9fzdi.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2838dfb02674c1e0fbbee1450e3e643f1110e1d9a272f3debf91fd0dad29f3a +size 288091 diff --git a/214032/episode_rosbags/episode_3_2024-12-10-21-43-33.rrd b/214032/episode_rosbags/episode_3_2024-12-10-21-43-33.rrd new file mode 100644 index 0000000000000000000000000000000000000000..e5384ff521f67e46020c94eece9caf40576109e2 --- /dev/null +++ b/214032/episode_rosbags/episode_3_2024-12-10-21-43-33.rrd @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f22437ca36530ce194bb2f80d9df1b41ab44c99975291a51576c5a4cff45f4e5 +size 428696522 diff --git a/snapshot_12499.pt b/snapshot_12499.pt new file mode 100644 index 0000000000000000000000000000000000000000..1f3d05b08b5daef7898bef0059edebbe96b493f0 --- /dev/null +++ b/snapshot_12499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cac2b77dd5220c4093e5f831629640cb84358cb1d60fb7c232fa30c9af5516a8 +size 910102360 diff --git a/snapshot_13249.pt b/snapshot_13249.pt new file mode 100644 index 0000000000000000000000000000000000000000..4393f8670a0b4dbce4f5183b5dbc1120c0b24ce1 --- /dev/null +++ b/snapshot_13249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbbec3e2c65539efab1674ff9e14d6b72432fa0588513b4ac86f2226da12171c +size 910102360 diff --git a/snapshot_13499.pt b/snapshot_13499.pt new file mode 100644 index 0000000000000000000000000000000000000000..b148d785d1c04a818c59e14e249a544502b8eb2a --- /dev/null +++ b/snapshot_13499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a593bb8eabe1283ee29f11856c8b836f87f45b7e7bd2728922fd4f50d2820fe +size 910102360 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..efef19a53c2eb36dc7d4100a2386822f4c83d2ec --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a77028d776839995e9df92b50ca81b4c45e7a4a6c0cb757a13479776ec96889 +size 910102360 diff --git a/snapshot_3000.pt b/snapshot_3000.pt new file mode 100644 index 0000000000000000000000000000000000000000..b4578e67d9d12795edeb705d5bda2837bbd740de --- /dev/null +++ b/snapshot_3000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0f9dd3c296080a0c5df65b8592de396ca6a6ec5e18ee88152b8223d25e97b47 +size 910102360 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..d1267f293575fb573414a20b5a0bf9777aaabb3b --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85cbcc554fb8e2c1e658a720e5e7c99610037590ac9bf3c40a540efd3318e9b9 +size 910102360 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..54df8638b343cb8beca57d731518bf8d756532af --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d4e9f8f5ef815dbab0b4af8b93617fcd2f5c5d45c11dfba9d1ff0b0235b3965 +size 910102360 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..4903797a4725aa36d2606e78242ee294403f3641 --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c11df9adeb7335d36d2f9f2354c21474af3cf2d4586b3dce92323c2017d0b1dc +size 910102360